Reorganising the repository - external renames and moves (#4074)

* Adding first rough ouline of the repository structure

* Remove old CI stuff

* add title

* formatting fixes

* move node-exits job's script to scripts dir

* Move docs into subdir

* move to bin

* move maintainence scripts, configs and helpers into its own dir

* add .local to ignore

* move core->client

* start up 'test' area

* move test client

* move test runtime

* make test move compile

* Add dependencies rule enforcement.

* Fix indexing.

* Update docs to reflect latest changes

* Moving /srml->/paint

* update docs

* move client/sr-* -> primitives/

* clean old readme

* remove old broken code in rhd

* update lock

* Step 1.

* starting to untangle client

* Fix after merge.

* start splitting out client interfaces

* move children and blockchain interfaces

* Move trie and state-machine to primitives.

* Fix WASM builds.

* fixing broken imports

* more interface moves

* move backend and light to interfaces

* move CallExecutor

* move cli off client

* moving around more interfaces

* re-add consensus crates into the mix

* fix subkey path

* relieve client from executor

* starting to pull out client from grandpa

* move is_decendent_of out of client

* grandpa still depends on client directly

* lemme tests pass

* rename srml->paint

* Make it compile.

* rename interfaces->client-api

* Move keyring to primitives.

* fixup libp2p dep

* fix broken use

* allow dependency enforcement to fail

* move fork-tree

* Moving wasm-builder

* make env

* move build-script-utils

* fixup broken crate depdencies and names

* fix imports for authority discovery

* fix typo

* update cargo.lock

* fixing imports

* Fix paths and add missing crates

* re-add missing crates

substrate/client/Cargo.toml

@@ -0,0 +1,41 @@
+[package]
+name = "substrate-client"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+client-api = { package = "substrate-client-api", path = "api" }
+block-builder = { package = "substrate-block-builder", path = "block-builder" }
+codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] }
+consensus = { package = "substrate-consensus-common", path = "../primitives/consensus/common" }
+kvdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
+derive_more = { version = "0.15.0" }
+executor = { package = "substrate-executor", path = "executor" }
+fnv = { version = "1.0.6" }
+futures = { version = "0.1.29" }
+futures03 = { package = "futures-preview", version = "0.3.0-alpha.19", features = ["compat"] }
+hash-db = { version = "0.15.2" }
+header-metadata = { package = "substrate-header-metadata", path = "header-metadata" }
+hex-literal = { version = "0.2.1" }
+inherents = { package = "substrate-inherents", path = "../primitives/inherents" }
+keyring = { package = "substrate-keyring", path = "../primitives/keyring" }
+log = { version = "0.4.8" }
+parking_lot = { version = "0.9.0" }
+primitives = { package = "substrate-primitives", path = "../primitives/core" }
+rstd = { package = "sr-std", path = "../primitives/sr-std" }
+runtime-version = { package = "sr-version", path = "../primitives/sr-version" }
+sr-api = { path = "../primitives/sr-api" }
+sr-primitives = { path = "../primitives/sr-primitives" }
+state-machine = { package = "substrate-state-machine", path = "../primitives/state-machine" }
+substrate-telemetry = { path = "telemetry" }
+trie = { package = "substrate-trie", path = "../primitives/trie" }
+
+[dev-dependencies]
+env_logger = "0.7.0"
+tempfile = "3.1.0"
+client-db = { package = "substrate-client-db", path = "./db", features = ["kvdb-rocksdb"] }
+test-client = { package = "substrate-test-runtime-client", path = "../test/utils/runtime/client" }
+kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
+panic-handler = { package = "substrate-panic-handler", path = "../primitives/panic-handler" }
+

substrate/client/api/Cargo.toml

@@ -0,0 +1,41 @@
+[package]
+name = "substrate-client-api"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+block-builder = { package = "substrate-block-builder", path = "../block-builder" }
+codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] }
+consensus = { package = "substrate-consensus-common", path = "../../primitives/consensus/common" }
+derive_more = { version = "0.15.0" }
+executor = { package = "substrate-executor", path = "../executor" }
+fnv = { version = "1.0.6" }
+futures = { version = "0.1.29" }
+futures03 = { package = "futures-preview", version = "0.3.0-alpha.19", features = ["compat"] }
+hash-db = { version = "0.15.2", default-features = false }
+header-metadata = { package = "substrate-header-metadata", path = "../header-metadata" }
+hex-literal = { version = "0.2.1" }
+inherents = { package = "substrate-inherents", path = "../../primitives/inherents", default-features = false }
+keyring = { package = "substrate-keyring", path = "../../primitives/keyring" }
+kvdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
+log = { version = "0.4.8" }
+parking_lot = { version = "0.9.0" }
+primitives = { package = "substrate-primitives", path = "../../primitives/core", default-features = false }
+rstd = { package = "sr-std", path = "../../primitives/sr-std", default-features = false }
+runtime-version = { package = "sr-version", path = "../../primitives/sr-version", default-features = false }
+sr-api = { path = "../../primitives/sr-api" }
+sr-primitives = { path = "../../primitives/sr-primitives", default-features = false }
+state-machine = { package = "substrate-state-machine", path = "../../primitives/state-machine" }
+substrate-telemetry = { path = "../telemetry" }
+trie = { package = "substrate-trie", path = "../../primitives/trie" }
+
+[dev-dependencies]
+env_logger = "0.7.0"
+tempfile = "3.1.0"
+client-db = { package = "substrate-client-db", path = ".././db", features = ["kvdb-rocksdb"] }
+test-primitives = { package = "substrate-test-primitives", path = "../../test/utils/primitives" }
+test-client = { package = "substrate-test-runtime-client", path = "../../test/utils/runtime/client" }
+kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
+panic-handler = { package = "substrate-panic-handler", path = "../../primitives/panic-handler" }
+

substrate/client/api/src/backend.rs

@@ -0,0 +1,336 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Substrate Client data backend
+
+use std::sync::Arc;
+use std::collections::HashMap;
+use primitives::ChangesTrieConfiguration;
+use sr_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay};
+use sr_primitives::traits::{Block as BlockT, NumberFor};
+use state_machine::backend::Backend as StateBackend;
+use state_machine::{ChangesTrieStorage as StateChangesTrieStorage, ChangesTrieTransaction};
+use crate::{
+	blockchain::{
+		Backend as BlockchainBackend, well_known_cache_keys
+	},
+	offchain::OffchainStorage,
+	error,
+	light::RemoteBlockchain,
+};
+use consensus::BlockOrigin;
+use hash_db::Hasher;
+use parking_lot::Mutex;
+
+/// In memory array of storage values.
+pub type StorageCollection = Vec<(Vec<u8>, Option<Vec<u8>>)>;
+
+/// In memory arrays of storage values for multiple child tries.
+pub type ChildStorageCollection = Vec<(Vec<u8>, StorageCollection)>;
+
+pub struct ImportSummary<Block: BlockT> {
+	pub hash: Block::Hash,
+	pub origin: BlockOrigin,
+	pub header: Block::Header,
+	pub is_new_best: bool,
+	pub storage_changes: Option<(StorageCollection, ChildStorageCollection)>,
+	pub retracted: Vec<Block::Hash>,
+}
+
+/// Import operation wrapper
+pub struct ClientImportOperation<
+	Block: BlockT,
+	H: Hasher<Out=Block::Hash>,
+	B: Backend<Block, H>,
+> {
+	pub op: B::BlockImportOperation,
+	pub notify_imported: Option<ImportSummary<Block>>,
+	pub notify_finalized: Vec<Block::Hash>,
+}
+
+/// State of a new block.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum NewBlockState {
+	/// Normal block.
+	Normal,
+	/// New best block.
+	Best,
+	/// Newly finalized block (implicitly best).
+	Final,
+}
+
+impl NewBlockState {
+	/// Whether this block is the new best block.
+	pub fn is_best(self) -> bool {
+		match self {
+			NewBlockState::Best | NewBlockState::Final => true,
+			NewBlockState::Normal => false,
+		}
+	}
+
+	/// Whether this block is considered final.
+	pub fn is_final(self) -> bool {
+		match self {
+			NewBlockState::Final => true,
+			NewBlockState::Best | NewBlockState::Normal => false,
+		}
+	}
+}
+
+/// Block insertion operation.
+///
+/// Keeps hold if the inserted block state and data.
+pub trait BlockImportOperation<Block, H> where
+	Block: BlockT,
+	H: Hasher<Out=Block::Hash>,
+{
+	/// Associated state backend type.
+	type State: StateBackend<H>;
+
+	/// Returns pending state.
+	///
+	/// Returns None for backends with locally-unavailable state data.
+	fn state(&self) -> error::Result<Option<&Self::State>>;
+
+	/// Append block data to the transaction.
+	fn set_block_data(
+		&mut self,
+		header: Block::Header,
+		body: Option<Vec<Block::Extrinsic>>,
+		justification: Option<Justification>,
+		state: NewBlockState,
+	) -> error::Result<()>;
+
+	/// Update cached data.
+	fn update_cache(&mut self, cache: HashMap<well_known_cache_keys::Id, Vec<u8>>);
+
+	/// Inject storage data into the database.
+	fn update_db_storage(&mut self, update: <Self::State as StateBackend<H>>::Transaction) -> error::Result<()>;
+
+	/// Inject storage data into the database replacing any existing data.
+	fn reset_storage(&mut self, top: StorageOverlay, children: ChildrenStorageOverlay) -> error::Result<H::Out>;
+
+	/// Set storage changes.
+	fn update_storage(
+		&mut self,
+		update: StorageCollection,
+		child_update: ChildStorageCollection,
+	) -> error::Result<()>;
+
+	/// Inject changes trie data into the database.
+	fn update_changes_trie(&mut self, update: ChangesTrieTransaction<H, NumberFor<Block>>) -> error::Result<()>;
+
+	/// Insert auxiliary keys.
+	///
+	/// Values are `None` if should be deleted.
+	fn insert_aux<I>(&mut self, ops: I) -> error::Result<()>
+		where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>;
+
+	/// Mark a block as finalized.
+	fn mark_finalized(&mut self, id: BlockId<Block>, justification: Option<Justification>) -> error::Result<()>;
+	/// Mark a block as new head. If both block import and set head are specified, set head overrides block import's best block rule.
+	fn mark_head(&mut self, id: BlockId<Block>) -> error::Result<()>;
+}
+
+/// Finalize Facilities
+pub trait Finalizer<Block: BlockT, H: Hasher<Out=Block::Hash>, B: Backend<Block, H>> {
+	/// Mark all blocks up to given as finalized in operation.
+	///
+	/// If `justification` is provided it is stored with the given finalized
+	/// block (any other finalized blocks are left unjustified).
+	///
+	/// If the block being finalized is on a different fork from the current
+	/// best block the finalized block is set as best, this might be slightly
+	/// inaccurate (i.e. outdated). Usages that require determining an accurate
+	/// best block should use `SelectChain` instead of the client.
+	fn apply_finality(
+		&self,
+		operation: &mut ClientImportOperation<Block, H, B>,
+		id: BlockId<Block>,
+		justification: Option<Justification>,
+		notify: bool,
+	) -> error::Result<()>;
+
+
+	/// Finalize a block.
+	///
+	/// This will implicitly finalize all blocks up to it and
+	/// fire finality notifications.
+	///
+	/// If the block being finalized is on a different fork from the current
+	/// best block, the finalized block is set as best. This might be slightly
+	/// inaccurate (i.e. outdated). Usages that require determining an accurate
+	/// best block should use `SelectChain` instead of the client.
+	///
+	/// Pass a flag to indicate whether finality notifications should be propagated.
+	/// This is usually tied to some synchronization state, where we don't send notifications
+	/// while performing major synchronization work.
+	fn finalize_block(
+		&self,
+		id: BlockId<Block>,
+		justification: Option<Justification>,
+		notify: bool,
+	) -> error::Result<()>;
+
+}
+
+/// Provides access to an auxiliary database.
+pub trait AuxStore {
+	/// Insert auxiliary data into key-value store.
+	///
+	/// Deletions occur after insertions.
+	fn insert_aux<
+		'a,
+		'b: 'a,
+		'c: 'a,
+		I: IntoIterator<Item=&'a(&'c [u8], &'c [u8])>,
+		D: IntoIterator<Item=&'a &'b [u8]>,
+	>(&self, insert: I, delete: D) -> error::Result<()>;
+
+	/// Query auxiliary data from key-value store.
+	fn get_aux(&self, key: &[u8]) -> error::Result<Option<Vec<u8>>>;
+}
+
+/// Client backend.
+///
+/// Manages the data layer.
+///
+/// Note on state pruning: while an object from `state_at` is alive, the state
+/// should not be pruned. The backend should internally reference-count
+/// its state objects.
+///
+/// The same applies for live `BlockImportOperation`s: while an import operation building on a parent `P`
+/// is alive, the state for `P` should not be pruned.
+pub trait Backend<Block, H>: AuxStore + Send + Sync where
+	Block: BlockT,
+	H: Hasher<Out=Block::Hash>,
+{
+	/// Associated block insertion operation type.
+	type BlockImportOperation: BlockImportOperation<Block, H, State=Self::State>;
+	/// Associated blockchain backend type.
+	type Blockchain: BlockchainBackend<Block>;
+	/// Associated state backend type.
+	type State: StateBackend<H>;
+	/// Changes trie storage.
+	type ChangesTrieStorage: PrunableStateChangesTrieStorage<Block, H>;
+	/// Offchain workers local storage.
+	type OffchainStorage: OffchainStorage;
+
+	/// Begin a new block insertion transaction with given parent block id.
+	///
+	/// When constructing the genesis, this is called with all-zero hash.
+	fn begin_operation(&self) -> error::Result<Self::BlockImportOperation>;
+
+	/// Note an operation to contain state transition.
+	fn begin_state_operation(&self, operation: &mut Self::BlockImportOperation, block: BlockId<Block>) -> error::Result<()>;
+
+	/// Commit block insertion.
+	fn commit_operation(&self, transaction: Self::BlockImportOperation) -> error::Result<()>;
+
+	/// Finalize block with given Id.
+	///
+	/// This should only be called if the parent of the given block has been finalized.
+	fn finalize_block(&self, block: BlockId<Block>, justification: Option<Justification>) -> error::Result<()>;
+
+	/// Returns reference to blockchain backend.
+	fn blockchain(&self) -> &Self::Blockchain;
+
+	/// Returns the used state cache, if existent.
+	fn used_state_cache_size(&self) -> Option<usize>;
+
+	/// Returns reference to changes trie storage.
+	fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage>;
+
+	/// Returns a handle to offchain storage.
+	fn offchain_storage(&self) -> Option<Self::OffchainStorage>;
+
+	/// Returns true if state for given block is available.
+	fn have_state_at(&self, hash: &Block::Hash, _number: NumberFor<Block>) -> bool {
+		self.state_at(BlockId::Hash(hash.clone())).is_ok()
+	}
+
+	/// Returns state backend with post-state of given block.
+	fn state_at(&self, block: BlockId<Block>) -> error::Result<Self::State>;
+
+	/// Destroy state and save any useful data, such as cache.
+	fn destroy_state(&self, _state: Self::State) -> error::Result<()> {
+		Ok(())
+	}
+
+	/// Attempts to revert the chain by `n` blocks.
+	///
+	/// Returns the number of blocks that were successfully reverted.
+	fn revert(&self, n: NumberFor<Block>) -> error::Result<NumberFor<Block>>;
+
+	/// Insert auxiliary data into key-value store.
+	fn insert_aux<
+		'a,
+		'b: 'a,
+		'c: 'a,
+		I: IntoIterator<Item=&'a(&'c [u8], &'c [u8])>,
+		D: IntoIterator<Item=&'a &'b [u8]>,
+	>(&self, insert: I, delete: D) -> error::Result<()>
+	{
+		AuxStore::insert_aux(self, insert, delete)
+	}
+	/// Query auxiliary data from key-value store.
+	fn get_aux(&self, key: &[u8]) -> error::Result<Option<Vec<u8>>> {
+		AuxStore::get_aux(self, key)
+	}
+
+	/// Gain access to the import lock around this backend.
+	///
+	/// _Note_ Backend isn't expected to acquire the lock by itself ever. Rather
+	/// the using components should acquire and hold the lock whenever they do
+	/// something that the import of a block would interfere with, e.g. importing
+	/// a new block or calculating the best head.
+	fn get_import_lock(&self) -> &Mutex<()>;
+}
+
+/// Changes trie storage that supports pruning.
+pub trait PrunableStateChangesTrieStorage<Block: BlockT, H: Hasher>:
+	StateChangesTrieStorage<H, NumberFor<Block>>
+{
+	/// Get number block of oldest, non-pruned changes trie.
+	fn oldest_changes_trie_block(
+		&self,
+		config: &ChangesTrieConfiguration,
+		best_finalized: NumberFor<Block>,
+	) -> NumberFor<Block>;
+}
+
+/// Mark for all Backend implementations, that are making use of state data, stored locally.
+pub trait LocalBackend<Block, H>: Backend<Block, H>
+where
+	Block: BlockT,
+	H: Hasher<Out=Block::Hash>,
+{}
+
+/// Mark for all Backend implementations, that are fetching required state data from remote nodes.
+pub trait RemoteBackend<Block, H>: Backend<Block, H>
+where
+	Block: BlockT,
+	H: Hasher<Out=Block::Hash>,
+{
+	/// Returns true if the state for given block is available locally.
+	fn is_local_state_available(&self, block: &BlockId<Block>) -> bool;
+
+	/// Returns reference to blockchain backend.
+	///
+	/// Returned backend either resolves blockchain data
+	/// locally, or prepares request to fetch that data from remote node.
+	fn remote_blockchain(&self) -> Arc<dyn RemoteBlockchain<Block>>;
+}

substrate/client/api/src/blockchain.rs

@@ -0,0 +1,275 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Substrate blockchain trait
+
+use std::sync::Arc;
+
+use sr_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor};
+use sr_primitives::generic::BlockId;
+use sr_primitives::Justification;
+use log::warn;
+use parking_lot::Mutex;
+
+use header_metadata::HeaderMetadata;
+
+use crate::error::{Error, Result};
+
+/// Blockchain database header backend. Does not perform any validation.
+pub trait HeaderBackend<Block: BlockT>: Send + Sync {
+	/// Get block header. Returns `None` if block is not found.
+	fn header(&self, id: BlockId<Block>) -> Result<Option<Block::Header>>;
+	/// Get blockchain info.
+	fn info(&self) -> Info<Block>;
+	/// Get block status.
+	fn status(&self, id: BlockId<Block>) -> Result<BlockStatus>;
+	/// Get block number by hash. Returns `None` if the header is not in the chain.
+	fn number(&self, hash: Block::Hash) -> Result<Option<<<Block as BlockT>::Header as HeaderT>::Number>>;
+	/// Get block hash by number. Returns `None` if the header is not in the chain.
+	fn hash(&self, number: NumberFor<Block>) -> Result<Option<Block::Hash>>;
+
+	/// Convert an arbitrary block ID into a block hash.
+	fn block_hash_from_id(&self, id: &BlockId<Block>) -> Result<Option<Block::Hash>> {
+		match *id {
+			BlockId::Hash(h) => Ok(Some(h)),
+			BlockId::Number(n) => self.hash(n),
+		}
+	}
+
+	/// Convert an arbitrary block ID into a block hash.
+	fn block_number_from_id(&self, id: &BlockId<Block>) -> Result<Option<NumberFor<Block>>> {
+		match *id {
+			BlockId::Hash(_) => Ok(self.header(*id)?.map(|h| h.number().clone())),
+			BlockId::Number(n) => Ok(Some(n)),
+		}
+	}
+
+	/// Get block header. Returns `UnknownBlock` error if block is not found.
+	fn expect_header(&self, id: BlockId<Block>) -> Result<Block::Header> {
+		self.header(id)?.ok_or_else(|| Error::UnknownBlock(format!("{}", id)))
+	}
+
+	/// Convert an arbitrary block ID into a block number. Returns `UnknownBlock` error if block is not found.
+	fn expect_block_number_from_id(&self, id: &BlockId<Block>) -> Result<NumberFor<Block>> {
+		self.block_number_from_id(id)
+			.and_then(|n| n.ok_or_else(|| Error::UnknownBlock(format!("{}", id))))
+	}
+
+	/// Convert an arbitrary block ID into a block hash. Returns `UnknownBlock` error if block is not found.
+	fn expect_block_hash_from_id(&self, id: &BlockId<Block>) -> Result<Block::Hash> {
+		self.block_hash_from_id(id)
+			.and_then(|n| n.ok_or_else(|| Error::UnknownBlock(format!("{}", id))))
+	}
+}
+
+/// Blockchain database backend. Does not perform any validation.
+pub trait Backend<Block: BlockT>: HeaderBackend<Block> + HeaderMetadata<Block, Error=Error> {
+	/// Get block body. Returns `None` if block is not found.
+	fn body(&self, id: BlockId<Block>) -> Result<Option<Vec<<Block as BlockT>::Extrinsic>>>;
+	/// Get block justification. Returns `None` if justification does not exist.
+	fn justification(&self, id: BlockId<Block>) -> Result<Option<Justification>>;
+	/// Get last finalized block hash.
+	fn last_finalized(&self) -> Result<Block::Hash>;
+	/// Returns data cache reference, if it is enabled on this backend.
+	fn cache(&self) -> Option<Arc<dyn Cache<Block>>>;
+
+	/// Returns hashes of all blocks that are leaves of the block tree.
+	/// in other words, that have no children, are chain heads.
+	/// Results must be ordered best (longest, highest) chain first.
+	fn leaves(&self) -> Result<Vec<Block::Hash>>;
+
+	/// Return hashes of all blocks that are children of the block with `parent_hash`.
+	fn children(&self, parent_hash: Block::Hash) -> Result<Vec<Block::Hash>>;
+
+	/// Get the most recent block hash of the best (longest) chains
+	/// that contain block with the given `target_hash`.
+	///
+	/// The search space is always limited to blocks which are in the finalized
+	/// chain or descendents of it.
+	///
+	/// If `maybe_max_block_number` is `Some(max_block_number)`
+	/// the search is limited to block `numbers <= max_block_number`.
+	/// in other words as if there were no blocks greater `max_block_number`.
+	/// Returns `Ok(None)` if `target_hash` is not found in search space.
+	/// TODO: document time complexity of this, see [#1444](https://github.com/paritytech/substrate/issues/1444)
+	fn best_containing(
+		&self,
+		target_hash: Block::Hash,
+		maybe_max_number: Option<NumberFor<Block>>,
+		import_lock: &Mutex<()>,
+	) -> Result<Option<Block::Hash>> {
+		let target_header = {
+			match self.header(BlockId::Hash(target_hash))? {
+				Some(x) => x,
+				// target not in blockchain
+				None => { return Ok(None); },
+			}
+		};
+
+		if let Some(max_number) = maybe_max_number {
+			// target outside search range
+			if target_header.number() > &max_number {
+				return Ok(None);
+			}
+		}
+
+		let leaves = {
+			// ensure no blocks are imported during this code block.
+			// an import could trigger a reorg which could change the canonical chain.
+			// we depend on the canonical chain staying the same during this code block.
+			let _import_guard = import_lock.lock();
+
+			let info = self.info();
+
+			// this can be `None` if the best chain is shorter than the target header.
+			let maybe_canon_hash = self.hash(*target_header.number())?;
+
+			if maybe_canon_hash.as_ref() == Some(&target_hash) {
+				// if a `max_number` is given we try to fetch the block at the
+				// given depth, if it doesn't exist or `max_number` is not
+				// provided, we continue to search from all leaves below.
+				if let Some(max_number) = maybe_max_number {
+					if let Some(header) = self.hash(max_number)? {
+						return Ok(Some(header));
+					}
+				}
+			} else if info.finalized_number >= *target_header.number() {
+				// header is on a dead fork.
+				return Ok(None);
+			}
+
+			self.leaves()?
+		};
+
+		// for each chain. longest chain first. shortest last
+		for leaf_hash in leaves {
+			// start at the leaf
+			let mut current_hash = leaf_hash;
+
+			// if search is not restricted then the leaf is the best
+			let mut best_hash = leaf_hash;
+
+			// go backwards entering the search space
+			// waiting until we are <= max_number
+			if let Some(max_number) = maybe_max_number {
+				loop {
+					let current_header = self.header(BlockId::Hash(current_hash.clone()))?
+						.ok_or_else(|| Error::from(format!("failed to get header for hash {}", current_hash)))?;
+
+					if current_header.number() <= &max_number {
+						best_hash = current_header.hash();
+						break;
+					}
+
+					current_hash = *current_header.parent_hash();
+				}
+			}
+
+			// go backwards through the chain (via parent links)
+			loop {
+				// until we find target
+				if current_hash == target_hash {
+					return Ok(Some(best_hash));
+				}
+
+				let current_header = self.header(BlockId::Hash(current_hash.clone()))?
+					.ok_or_else(|| Error::from(format!("failed to get header for hash {}", current_hash)))?;
+
+				// stop search in this chain once we go below the target's block number
+				if current_header.number() < target_header.number() {
+					break;
+				}
+
+				current_hash = *current_header.parent_hash();
+			}
+		}
+
+		// header may be on a dead fork -- the only leaves that are considered are
+		// those which can still be finalized.
+		//
+		// FIXME #1558 only issue this warning when not on a dead fork
+		warn!(
+			"Block {:?} exists in chain but not found when following all \
+			leaves backwards. Number limit = {:?}",
+			target_hash,
+			maybe_max_number,
+		);
+
+		Ok(None)
+	}
+}
+
+/// Provides access to the optional cache.
+pub trait ProvideCache<Block: BlockT> {
+	/// Returns data cache reference, if it is enabled on this backend.
+	fn cache(&self) -> Option<Arc<dyn Cache<Block>>>;
+}
+
+/// Blockchain optional data cache.
+pub trait Cache<Block: BlockT>: Send + Sync {
+	/// Initialize genesis value for the given cache.
+	///
+	/// The operation should be performed once before anything else is inserted in the cache.
+	/// Otherwise cache may end up in inconsistent state.
+	fn initialize(&self, key: &well_known_cache_keys::Id, value_at_genesis: Vec<u8>) -> Result<()>;
+	/// Returns cached value by the given key.
+	///
+	/// Returned tuple is the range where value has been active and the value itself.
+	fn get_at(
+		&self,
+		key: &well_known_cache_keys::Id,
+		block: &BlockId<Block>,
+	) -> Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)>;
+}
+
+/// Blockchain info
+#[derive(Debug)]
+pub struct Info<Block: BlockT> {
+	/// Best block hash.
+	pub best_hash: Block::Hash,
+	/// Best block number.
+	pub best_number: <<Block as BlockT>::Header as HeaderT>::Number,
+	/// Genesis block hash.
+	pub genesis_hash: Block::Hash,
+	/// The head of the finalized chain.
+	pub finalized_hash: Block::Hash,
+	/// Last finalized block number.
+	pub finalized_number: <<Block as BlockT>::Header as HeaderT>::Number,
+}
+
+/// Block status.
+#[derive(Debug, PartialEq, Eq)]
+pub enum BlockStatus {
+	/// Already in the blockchain.
+	InChain,
+	/// Not in the queue or the blockchain.
+	Unknown,
+}
+
+/// A list of all well known keys in the blockchain cache.
+pub mod well_known_cache_keys {
+	/// The type representing cache keys.
+	pub type Id = consensus::import_queue::CacheKeyId;
+
+	/// A list of authorities.
+	pub const AUTHORITIES: Id = *b"auth";
+
+	/// Current Epoch data.
+	pub const EPOCH: Id = *b"epch";
+
+	/// Changes trie configuration.
+	pub const CHANGES_TRIE_CONFIG: Id = *b"chtr";
+}

substrate/client/api/src/call_executor.rs

@@ -0,0 +1,148 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{cmp::Ord, panic::UnwindSafe, result, cell::RefCell};
+use codec::{Encode, Decode};
+use sr_primitives::{
+	generic::BlockId, traits::Block as BlockT, traits::NumberFor,
+};
+use state_machine::{
+	self, OverlayedChanges, ExecutionManager, ExecutionStrategy,
+	ChangesTrieTransaction, StorageProof,
+};
+use executor::{RuntimeVersion, NativeVersion};
+use hash_db::Hasher;
+use primitives::{offchain::OffchainExt, Blake2Hasher, NativeOrEncoded};
+
+use sr_api::{ProofRecorder, InitializeBlock};
+use crate::error;
+
+/// Method call executor.
+pub trait CallExecutor<B, H>
+where
+	B: BlockT,
+	H: Hasher<Out=B::Hash>,
+	H::Out: Ord,
+{
+	/// Externalities error type.
+	type Error: state_machine::Error;
+
+	/// Execute a call to a contract on top of state in a block of given hash.
+	///
+	/// No changes are made.
+	fn call(
+		&self,
+		id: &BlockId<B>,
+		method: &str,
+		call_data: &[u8],
+		strategy: ExecutionStrategy,
+		side_effects_handler: Option<OffchainExt>,
+	) -> Result<Vec<u8>, error::Error>;
+
+	/// Execute a contextual call on top of state in a block of a given hash.
+	///
+	/// No changes are made.
+	/// Before executing the method, passed header is installed as the current header
+	/// of the execution context.
+	fn contextual_call<
+		'a,
+		IB: Fn() -> error::Result<()>,
+		EM: Fn(
+			Result<NativeOrEncoded<R>, Self::Error>,
+			Result<NativeOrEncoded<R>, Self::Error>
+		) -> Result<NativeOrEncoded<R>, Self::Error>,
+		R: Encode + Decode + PartialEq,
+		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
+	>(
+		&self,
+		initialize_block_fn: IB,
+		at: &BlockId<B>,
+		method: &str,
+		call_data: &[u8],
+		changes: &RefCell<OverlayedChanges>,
+		initialize_block: InitializeBlock<'a, B>,
+		execution_manager: ExecutionManager<EM>,
+		native_call: Option<NC>,
+		side_effects_handler: Option<OffchainExt>,
+		proof_recorder: &Option<ProofRecorder<B>>,
+		enable_keystore: bool,
+	) -> error::Result<NativeOrEncoded<R>> where ExecutionManager<EM>: Clone;
+
+	/// Extract RuntimeVersion of given block
+	///
+	/// No changes are made.
+	fn runtime_version(&self, id: &BlockId<B>) -> Result<RuntimeVersion, error::Error>;
+
+	/// Execute a call to a contract on top of given state.
+	///
+	/// No changes are made.
+	fn call_at_state<
+		S: state_machine::Backend<H>,
+		F: FnOnce(
+			Result<NativeOrEncoded<R>, Self::Error>,
+			Result<NativeOrEncoded<R>, Self::Error>,
+		) -> Result<NativeOrEncoded<R>, Self::Error>,
+		R: Encode + Decode + PartialEq,
+		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
+	>(&self,
+		state: &S,
+		overlay: &mut OverlayedChanges,
+		method: &str,
+		call_data: &[u8],
+		manager: ExecutionManager<F>,
+		native_call: Option<NC>,
+		side_effects_handler: Option<OffchainExt>,
+	) -> Result<
+		(
+			NativeOrEncoded<R>,
+			(S::Transaction, H::Out),
+			Option<ChangesTrieTransaction<Blake2Hasher, NumberFor<B>>>
+		),
+		error::Error,
+	>;
+
+	/// Execute a call to a contract on top of given state, gathering execution proof.
+	///
+	/// No changes are made.
+	fn prove_at_state<S: state_machine::Backend<H>>(
+		&self,
+		mut state: S,
+		overlay: &mut OverlayedChanges,
+		method: &str,
+		call_data: &[u8]
+	) -> Result<(Vec<u8>, StorageProof), error::Error> {
+		let trie_state = state.as_trie_backend()
+			.ok_or_else(||
+				Box::new(state_machine::ExecutionError::UnableToGenerateProof)
+					as Box<dyn state_machine::Error>
+			)?;
+		self.prove_at_trie_state(trie_state, overlay, method, call_data)
+	}
+
+	/// Execute a call to a contract on top of given trie state, gathering execution proof.
+	///
+	/// No changes are made.
+	fn prove_at_trie_state<S: state_machine::TrieBackendStorage<H>>(
+		&self,
+		trie_state: &state_machine::TrieBackend<S, H>,
+		overlay: &mut OverlayedChanges,
+		method: &str,
+		call_data: &[u8]
+	) -> Result<(Vec<u8>, StorageProof), error::Error>;
+
+	/// Get runtime version if supported.
+	fn native_runtime_version(&self) -> Option<&NativeVersion>;
+}

substrate/client/api/src/client.rs

@@ -0,0 +1,141 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::HashMap;
+use futures03::channel::mpsc;
+use primitives::storage::StorageKey;
+use state_machine::ExecutionStrategy;
+use sr_primitives::{
+    traits::{Block as BlockT, NumberFor},
+    generic::BlockId
+};
+use consensus::BlockOrigin;
+
+use crate::blockchain::Info;
+use crate::notifications::StorageEventStream;
+use crate::error;
+
+/// Type that implements `futures::Stream` of block import events.
+pub type ImportNotifications<Block> = mpsc::UnboundedReceiver<BlockImportNotification<Block>>;
+
+/// A stream of block finality notifications.
+pub type FinalityNotifications<Block> = mpsc::UnboundedReceiver<FinalityNotification<Block>>;
+
+/// Expected hashes of blocks at given heights.
+///
+/// This may be used as chain spec extension to filter out known, unwanted forks.
+pub type ForkBlocks<Block> = Option<HashMap<NumberFor<Block>, <Block as BlockT>::Hash>>;
+
+/// Execution strategies settings.
+#[derive(Debug, Clone)]
+pub struct ExecutionStrategies {
+	/// Execution strategy used when syncing.
+	pub syncing: ExecutionStrategy,
+	/// Execution strategy used when importing blocks.
+	pub importing: ExecutionStrategy,
+	/// Execution strategy used when constructing blocks.
+	pub block_construction: ExecutionStrategy,
+	/// Execution strategy used for offchain workers.
+	pub offchain_worker: ExecutionStrategy,
+	/// Execution strategy used in other cases.
+	pub other: ExecutionStrategy,
+}
+
+impl Default for ExecutionStrategies {
+	fn default() -> ExecutionStrategies {
+		ExecutionStrategies {
+			syncing: ExecutionStrategy::NativeElseWasm,
+			importing: ExecutionStrategy::NativeElseWasm,
+			block_construction: ExecutionStrategy::AlwaysWasm,
+			offchain_worker: ExecutionStrategy::NativeWhenPossible,
+			other: ExecutionStrategy::NativeElseWasm,
+		}
+	}
+}
+
+/// Figure out the block type for a given type (for now, just a `Client`).
+pub trait BlockOf {
+	/// The type of the block.
+	type Type: BlockT;
+}
+
+/// A source of blockchain events.
+pub trait BlockchainEvents<Block: BlockT> {
+	/// Get block import event stream. Not guaranteed to be fired for every
+	/// imported block.
+	fn import_notification_stream(&self) -> ImportNotifications<Block>;
+
+	/// Get a stream of finality notifications. Not guaranteed to be fired for every
+	/// finalized block.
+	fn finality_notification_stream(&self) -> FinalityNotifications<Block>;
+
+	/// Get storage changes event stream.
+	///
+	/// Passing `None` as `filter_keys` subscribes to all storage changes.
+	fn storage_changes_notification_stream(
+		&self,
+		filter_keys: Option<&[StorageKey]>,
+		child_filter_keys: Option<&[(StorageKey, Option<Vec<StorageKey>>)]>,
+	) -> error::Result<StorageEventStream<Block::Hash>>;
+}
+
+/// Fetch block body by ID.
+pub trait BlockBody<Block: BlockT> {
+	/// Get block body by ID. Returns `None` if the body is not stored.
+	fn block_body(&self,
+		id: &BlockId<Block>
+	) -> error::Result<Option<Vec<<Block as BlockT>::Extrinsic>>>;
+}
+
+/// Provide a list of potential uncle headers for a given block.
+pub trait ProvideUncles<Block: BlockT> {
+	/// Gets the uncles of the block with `target_hash` going back `max_generation` ancestors.
+	fn uncles(&self, target_hash: Block::Hash, max_generation: NumberFor<Block>)
+		-> error::Result<Vec<Block::Header>>;
+}
+
+/// Client info
+#[derive(Debug)]
+pub struct ClientInfo<Block: BlockT> {
+	/// Best block hash.
+	pub chain: Info<Block>,
+	/// State Cache Size currently used by the backend
+	pub used_state_cache_size: Option<usize>,
+}
+
+/// Summary of an imported block
+#[derive(Clone, Debug)]
+pub struct BlockImportNotification<Block: BlockT> {
+	/// Imported block header hash.
+	pub hash: Block::Hash,
+	/// Imported block origin.
+	pub origin: BlockOrigin,
+	/// Imported block header.
+	pub header: Block::Header,
+	/// Is this the new best block.
+	pub is_new_best: bool,
+	/// List of retracted blocks ordered by block number.
+	pub retracted: Vec<Block::Hash>,
+}
+
+/// Summary of a finalized block.
+#[derive(Clone, Debug)]
+pub struct FinalityNotification<Block: BlockT> {
+	/// Imported block header hash.
+	pub hash: Block::Hash,
+	/// Imported block header.
+	pub header: Block::Header,
+}

substrate/client/api/src/error.rs

@@ -0,0 +1,143 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Substrate client possible errors.
+
+use std::{self, error, result};
+use state_machine;
+use sr_primitives::ApplyError;
+use consensus;
+use derive_more::{Display, From};
+
+/// Client Result type alias
+pub type Result<T> = result::Result<T, Error>;
+
+/// Substrate Client error
+#[derive(Debug, Display, From)]
+pub enum Error {
+	/// Consensus Error
+	#[display(fmt = "Consensus: {}", _0)]
+	Consensus(consensus::Error),
+	/// Backend error.
+	#[display(fmt = "Backend error: {}", _0)]
+	Backend(String),
+	/// Unknown block.
+	#[display(fmt = "UnknownBlock: {}", _0)]
+	UnknownBlock(String),
+	/// Applying extrinsic error.
+	#[display(fmt = "Extrinsic error: {:?}", _0)]
+	ApplyExtrinsicFailed(ApplyError),
+	/// Execution error.
+	#[display(fmt = "Execution: {}", _0)]
+	Execution(Box<dyn state_machine::Error>),
+	/// Blockchain error.
+	#[display(fmt = "Blockchain: {}", _0)]
+	Blockchain(Box<Error>),
+	/// Invalid authorities set received from the runtime.
+	#[display(fmt = "Current state of blockchain has invalid authorities set")]
+	InvalidAuthoritiesSet,
+	/// Could not get runtime version.
+	#[display(fmt = "On-chain runtime does not specify version")]
+	VersionInvalid,
+	/// Genesis config is invalid.
+	#[display(fmt = "Genesis config provided is invalid")]
+	GenesisInvalid,
+	/// Error decoding header justification.
+	#[display(fmt = "error decoding justification for header")]
+	JustificationDecode,
+	/// Justification for header is correctly encoded, but invalid.
+	#[display(fmt = "bad justification for header: {}", _0)]
+	BadJustification(String),
+	/// Not available on light client.
+	#[display(fmt = "This method is not currently available when running in light client mode")]
+	NotAvailableOnLightClient,
+	/// Invalid remote CHT-based proof.
+	#[display(fmt = "Remote node has responded with invalid header proof")]
+	InvalidCHTProof,
+	/// Remote fetch has been cancelled.
+	#[display(fmt = "Remote data fetch has been cancelled")]
+	RemoteFetchCancelled,
+	/// Remote fetch has been failed.
+	#[display(fmt = "Remote data fetch has been failed")]
+	RemoteFetchFailed,
+	/// Error decoding call result.
+	#[display(fmt = "Error decoding call result of {}: {}", _0, _1)]
+	CallResultDecode(&'static str, codec::Error),
+	/// Error converting a parameter between runtime and node.
+	#[display(fmt = "Error converting `{}` between runtime and node", _0)]
+	RuntimeParamConversion(String),
+	/// Changes tries are not supported.
+	#[display(fmt = "Changes tries are not supported by the runtime")]
+	ChangesTriesNotSupported,
+	/// Key changes query has failed.
+	#[display(fmt = "Failed to check changes proof: {}", _0)]
+	ChangesTrieAccessFailed(String),
+	/// Last finalized block not parent of current.
+	#[display(fmt = "Did not finalize blocks in sequential order.")]
+	NonSequentialFinalization(String),
+	/// Safety violation: new best block not descendent of last finalized.
+	#[display(fmt = "Potential long-range attack: block not in finalized chain.")]
+	NotInFinalizedChain,
+	/// Hash that is required for building CHT is missing.
+	#[display(fmt = "Failed to get hash of block for building CHT")]
+	MissingHashRequiredForCHT,
+	/// Invalid calculated state root on block import.
+	#[display(fmt = "Calculated state root does not match.")]
+	InvalidStateRoot,
+	/// A convenience variant for String
+	#[display(fmt = "{}", _0)]
+	Msg(String),
+}
+
+impl error::Error for Error {
+	fn source(&self) -> Option<&(dyn error::Error + 'static)> {
+		match self {
+			Error::Consensus(e) => Some(e),
+			Error::Blockchain(e) => Some(e),
+			_ => None,
+		}
+	}
+}
+
+impl From<String> for Error {
+	fn from(s: String) -> Self {
+		Error::Msg(s)
+	}
+}
+
+impl<'a> From<&'a str> for Error {
+	fn from(s: &'a str) -> Self {
+		Error::Msg(s.into())
+	}
+}
+
+impl From<block_builder::ApplyExtrinsicFailed> for Error {
+	fn from(err: block_builder::ApplyExtrinsicFailed) -> Self {
+		Self::ApplyExtrinsicFailed(err.0)
+	}
+}
+
+impl Error {
+	/// Chain a blockchain error.
+	pub fn from_blockchain(e: Box<Error>) -> Self {
+		Error::Blockchain(e)
+	}
+
+	/// Chain a state error.
+	pub fn from_state(e: Box<dyn state_machine::Error>) -> Self {
+		Error::Execution(e)
+	}
+}

substrate/client/api/src/lib.rs

@@ -0,0 +1,84 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Substrate client interfaces.
+
+// TODO: make internal
+pub mod error;
+pub mod backend;
+pub mod blockchain;
+pub mod light;
+pub mod notifications;
+pub mod call_executor;
+pub mod client;
+pub mod offchain;
+
+pub use error::*;
+pub use backend::*;
+pub use blockchain::*;
+pub use light::*;
+pub use notifications::*;
+pub use call_executor::*;
+pub use offchain::*;
+pub use client::*;
+
+pub use state_machine::{StorageProof, ExecutionStrategy};
+
+
+/// Utility methods for the client.
+pub mod utils {
+    use super::HeaderBackend;
+    use header_metadata::HeaderMetadata;
+	use crate::error;
+    use primitives::H256;
+	use sr_primitives::traits::{Block as BlockT};
+	use std::borrow::Borrow;
+
+	/// Returns a function for checking block ancestry, the returned function will
+	/// return `true` if the given hash (second parameter) is a descendent of the
+	/// base (first parameter). If the `current` parameter is defined, it should
+	/// represent the current block `hash` and its `parent hash`, if given the
+	/// function that's returned will assume that `hash` isn't part of the local DB
+	/// yet, and all searches in the DB will instead reference the parent.
+	pub fn is_descendent_of<'a, Block: BlockT<Hash=H256>, T, H: Borrow<H256> + 'a>(
+		client: &'a T,
+		current: Option<(H, H)>,
+	) -> impl Fn(&H256, &H256) -> Result<bool, error::Error> + 'a
+		where T: HeaderBackend<Block> + HeaderMetadata<Block, Error=error::Error>,
+	{
+		move |base, hash| {
+			if base == hash { return Ok(false); }
+
+			let current = current.as_ref().map(|(c, p)| (c.borrow(), p.borrow()));
+
+			let mut hash = hash;
+			if let Some((current_hash, current_parent_hash)) = current {
+				if base == current_hash { return Ok(false); }
+				if hash == current_hash {
+					if base == current_parent_hash {
+						return Ok(true);
+					} else {
+						hash = current_parent_hash;
+					}
+				}
+			}
+
+			let ancestor = header_metadata::lowest_common_ancestor(client, *hash, *base)?;
+
+			Ok(ancestor.hash == *base)
+		}
+	}
+}

substrate/client/api/src/light.rs

@@ -0,0 +1,335 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Substrate light client interfaces
+
+use std::sync::Arc;
+use std::collections::{BTreeMap, HashMap};
+use std::future::Future;
+
+use sr_primitives::{
+    traits::{
+        Block as BlockT, Header as HeaderT, NumberFor,
+    },
+    generic::BlockId
+};
+use primitives::{ChangesTrieConfiguration};
+use state_machine::StorageProof;
+use header_metadata::HeaderMetadata;
+use crate::{
+	backend::{
+		AuxStore, NewBlockState,
+	},
+    blockchain::{
+        well_known_cache_keys, HeaderBackend, Cache as BlockchainCache,
+    },
+    error::{ Error as ClientError, Result as ClientResult },
+};
+/// Remote call request.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct RemoteCallRequest<Header: HeaderT> {
+	/// Call at state of given block.
+	pub block: Header::Hash,
+	/// Header of block at which call is performed.
+	pub header: Header,
+	/// Method to call.
+	pub method: String,
+	/// Call data.
+	pub call_data: Vec<u8>,
+	/// Number of times to retry request. None means that default RETRY_COUNT is used.
+	pub retry_count: Option<usize>,
+}
+
+/// Remote canonical header request.
+#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
+pub struct RemoteHeaderRequest<Header: HeaderT> {
+	/// The root of CHT this block is included in.
+	pub cht_root: Header::Hash,
+	/// Number of the header to query.
+	pub block: Header::Number,
+	/// Number of times to retry request. None means that default RETRY_COUNT is used.
+	pub retry_count: Option<usize>,
+}
+
+/// Remote storage read request.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct RemoteReadRequest<Header: HeaderT> {
+	/// Read at state of given block.
+	pub block: Header::Hash,
+	/// Header of block at which read is performed.
+	pub header: Header,
+	/// Storage key to read.
+	pub keys: Vec<Vec<u8>>,
+	/// Number of times to retry request. None means that default RETRY_COUNT is used.
+	pub retry_count: Option<usize>,
+}
+
+/// Remote storage read child request.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct RemoteReadChildRequest<Header: HeaderT> {
+	/// Read at state of given block.
+	pub block: Header::Hash,
+	/// Header of block at which read is performed.
+	pub header: Header,
+	/// Storage key for child.
+	pub storage_key: Vec<u8>,
+	/// Child storage key to read.
+	pub keys: Vec<Vec<u8>>,
+	/// Number of times to retry request. None means that default RETRY_COUNT is used.
+	pub retry_count: Option<usize>,
+}
+
+/// Remote key changes read request.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct RemoteChangesRequest<Header: HeaderT> {
+	/// Changes trie configuration.
+	pub changes_trie_config: ChangesTrieConfiguration,
+	/// Query changes from range of blocks, starting (and including) with this hash...
+	pub first_block: (Header::Number, Header::Hash),
+	/// ...ending (and including) with this hash. Should come after first_block and
+	/// be the part of the same fork.
+	pub last_block: (Header::Number, Header::Hash),
+	/// Only use digests from blocks up to this hash. Should be last_block OR come
+	/// after this block and be the part of the same fork.
+	pub max_block: (Header::Number, Header::Hash),
+	/// Known changes trie roots for the range of blocks [tries_roots.0..max_block].
+	/// Proofs for roots of ascendants of tries_roots.0 are provided by the remote node.
+	pub tries_roots: (Header::Number, Header::Hash, Vec<Header::Hash>),
+	/// Optional Child Storage key to read.
+	pub storage_key: Option<Vec<u8>>,
+	/// Storage key to read.
+	pub key: Vec<u8>,
+	/// Number of times to retry request. None means that default RETRY_COUNT is used.
+	pub retry_count: Option<usize>,
+}
+
+/// Key changes read proof.
+#[derive(Debug, PartialEq, Eq)]
+pub struct ChangesProof<Header: HeaderT> {
+	/// Max block that has been used in changes query.
+	pub max_block: Header::Number,
+	/// All touched nodes of all changes tries.
+	pub proof: Vec<Vec<u8>>,
+	/// All changes tries roots that have been touched AND are missing from
+	/// the requester' node. It is a map of block number => changes trie root.
+	pub roots: BTreeMap<Header::Number, Header::Hash>,
+	/// The proofs for all changes tries roots that have been touched AND are
+	/// missing from the requester' node. It is a map of CHT number => proof.
+	pub roots_proof: StorageProof,
+}
+
+/// Remote block body request
+#[derive(Clone, Default, Debug, PartialEq, Eq, Hash)]
+pub struct RemoteBodyRequest<Header: HeaderT> {
+	/// Header of the requested block body
+	pub header: Header,
+	/// Number of times to retry request. None means that default RETRY_COUNT is used.
+	pub retry_count: Option<usize>,
+}
+
+/// Light client data fetcher. Implementations of this trait must check if remote data
+/// is correct (see FetchedDataChecker) and return already checked data.
+pub trait Fetcher<Block: BlockT>: Send + Sync {
+	/// Remote header future.
+	type RemoteHeaderResult: Future<Output = Result<Block::Header, ClientError>> + Send + 'static;
+	/// Remote storage read future.
+	type RemoteReadResult: Future<Output = Result<HashMap<Vec<u8>, Option<Vec<u8>>>, ClientError>> + Send + 'static;
+	/// Remote call result future.
+	type RemoteCallResult: Future<Output = Result<Vec<u8>, ClientError>> + Send + 'static;
+	/// Remote changes result future.
+	type RemoteChangesResult: Future<Output = Result<Vec<(NumberFor<Block>, u32)>, ClientError>> + Send + 'static;
+	/// Remote block body result future.
+	type RemoteBodyResult: Future<Output = Result<Vec<Block::Extrinsic>, ClientError>> + Send + 'static;
+
+	/// Fetch remote header.
+	fn remote_header(&self, request: RemoteHeaderRequest<Block::Header>) -> Self::RemoteHeaderResult;
+	/// Fetch remote storage value.
+	fn remote_read(
+		&self,
+		request: RemoteReadRequest<Block::Header>
+	) -> Self::RemoteReadResult;
+	/// Fetch remote storage child value.
+	fn remote_read_child(
+		&self,
+		request: RemoteReadChildRequest<Block::Header>
+	) -> Self::RemoteReadResult;
+	/// Fetch remote call result.
+	fn remote_call(&self, request: RemoteCallRequest<Block::Header>) -> Self::RemoteCallResult;
+	/// Fetch remote changes ((block number, extrinsic index)) where given key has been changed
+	/// at a given blocks range.
+	fn remote_changes(&self, request: RemoteChangesRequest<Block::Header>) -> Self::RemoteChangesResult;
+	/// Fetch remote block body
+	fn remote_body(&self, request: RemoteBodyRequest<Block::Header>) -> Self::RemoteBodyResult;
+}
+
+/// Light client remote data checker.
+///
+/// Implementations of this trait should not use any prunable blockchain data
+/// except that is passed to its methods.
+pub trait FetchChecker<Block: BlockT>: Send + Sync {
+	/// Check remote header proof.
+	fn check_header_proof(
+		&self,
+		request: &RemoteHeaderRequest<Block::Header>,
+		header: Option<Block::Header>,
+		remote_proof: StorageProof,
+	) -> ClientResult<Block::Header>;
+	/// Check remote storage read proof.
+	fn check_read_proof(
+		&self,
+		request: &RemoteReadRequest<Block::Header>,
+		remote_proof: StorageProof,
+	) -> ClientResult<HashMap<Vec<u8>, Option<Vec<u8>>>>;
+	/// Check remote storage read proof.
+	fn check_read_child_proof(
+		&self,
+		request: &RemoteReadChildRequest<Block::Header>,
+		remote_proof: StorageProof,
+	) -> ClientResult<HashMap<Vec<u8>, Option<Vec<u8>>>>;
+	/// Check remote method execution proof.
+	fn check_execution_proof(
+		&self,
+		request: &RemoteCallRequest<Block::Header>,
+		remote_proof: StorageProof,
+	) -> ClientResult<Vec<u8>>;
+	/// Check remote changes query proof.
+	fn check_changes_proof(
+		&self,
+		request: &RemoteChangesRequest<Block::Header>,
+		proof: ChangesProof<Block::Header>
+	) -> ClientResult<Vec<(NumberFor<Block>, u32)>>;
+	/// Check remote body proof.
+	fn check_body_proof(
+		&self,
+		request: &RemoteBodyRequest<Block::Header>,
+		body: Vec<Block::Extrinsic>
+	) -> ClientResult<Vec<Block::Extrinsic>>;
+}
+
+
+/// Light client blockchain storage.
+pub trait Storage<Block: BlockT>: AuxStore + HeaderBackend<Block> + HeaderMetadata<Block, Error=ClientError> {
+	/// Store new header. Should refuse to revert any finalized blocks.
+	///
+	/// Takes new authorities, the leaf state of the new block, and
+	/// any auxiliary storage updates to place in the same operation.
+	fn import_header(
+		&self,
+		header: Block::Header,
+		cache: HashMap<well_known_cache_keys::Id, Vec<u8>>,
+		state: NewBlockState,
+		aux_ops: Vec<(Vec<u8>, Option<Vec<u8>>)>,
+	) -> ClientResult<()>;
+
+	/// Set an existing block as new best block.
+	fn set_head(&self, block: BlockId<Block>) -> ClientResult<()>;
+
+	/// Mark historic header as finalized.
+	fn finalize_header(&self, block: BlockId<Block>) -> ClientResult<()>;
+
+	/// Get last finalized header.
+	fn last_finalized(&self) -> ClientResult<Block::Hash>;
+
+	/// Get headers CHT root for given block. Fails if the block is not pruned (not a part of any CHT).
+	fn header_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> ClientResult<Block::Hash>;
+
+	/// Get changes trie CHT root for given block. Fails if the block is not pruned (not a part of any CHT).
+	fn changes_trie_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> ClientResult<Block::Hash>;
+
+	/// Get storage cache.
+	fn cache(&self) -> Option<Arc<dyn BlockchainCache<Block>>>;
+}
+
+/// Remote header.
+#[derive(Debug)]
+pub enum LocalOrRemote<Data, Request> {
+	/// When data is available locally, it is returned.
+	Local(Data),
+	/// When data is unavailable locally, the request to fetch it from remote node is returned.
+	Remote(Request),
+	/// When data is unknown.
+	Unknown,
+}
+
+/// Futures-based blockchain backend that either resolves blockchain data
+/// locally, or fetches required data from remote node.
+pub trait RemoteBlockchain<Block: BlockT>: Send + Sync {
+	/// Get block header.
+	fn header(&self, id: BlockId<Block>) -> ClientResult<LocalOrRemote<
+		Block::Header,
+		RemoteHeaderRequest<Block::Header>,
+	>>;
+}
+
+
+
+#[cfg(test)]
+pub mod tests {
+	use futures03::future::Ready;
+	use parking_lot::Mutex;
+    use crate::error::Error as ClientError;
+    use test_primitives::{Block, Header, Extrinsic};
+	use super::*;
+
+	pub type OkCallFetcher = Mutex<Vec<u8>>;
+
+	fn not_implemented_in_tests<T, E>() -> Ready<Result<T, E>>
+	where
+		E: std::convert::From<&'static str>,
+	{
+		futures03::future::ready(Err("Not implemented on test node".into()))
+	}
+
+	impl Fetcher<Block> for OkCallFetcher {
+		type RemoteHeaderResult = Ready<Result<Header, ClientError>>;
+		type RemoteReadResult = Ready<Result<HashMap<Vec<u8>, Option<Vec<u8>>>, ClientError>>;
+		type RemoteCallResult = Ready<Result<Vec<u8>, ClientError>>;
+		type RemoteChangesResult = Ready<Result<Vec<(NumberFor<Block>, u32)>, ClientError>>;
+		type RemoteBodyResult = Ready<Result<Vec<Extrinsic>, ClientError>>;
+
+		fn remote_header(&self, _request: RemoteHeaderRequest<Header>) -> Self::RemoteHeaderResult {
+			not_implemented_in_tests()
+		}
+
+		fn remote_read(&self, _request: RemoteReadRequest<Header>) -> Self::RemoteReadResult {
+			not_implemented_in_tests()
+		}
+
+		fn remote_read_child(&self, _request: RemoteReadChildRequest<Header>) -> Self::RemoteReadResult {
+			not_implemented_in_tests()
+		}
+
+		fn remote_call(&self, _request: RemoteCallRequest<Header>) -> Self::RemoteCallResult {
+			futures03::future::ready(Ok((*self.lock()).clone()))
+		}
+
+		fn remote_changes(&self, _request: RemoteChangesRequest<Header>) -> Self::RemoteChangesResult {
+			not_implemented_in_tests()
+		}
+
+		fn remote_body(&self, _request: RemoteBodyRequest<Header>) -> Self::RemoteBodyResult {
+			not_implemented_in_tests()
+		}
+	}
+}

substrate/client/api/src/notifications.rs

@@ -0,0 +1,480 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Storage notifications
+
+use std::{
+	collections::{HashSet, HashMap},
+	sync::Arc,
+};
+
+use fnv::{FnvHashSet, FnvHashMap};
+use futures03::channel::mpsc;
+use primitives::storage::{StorageKey, StorageData};
+use sr_primitives::traits::Block as BlockT;
+
+/// Storage change set
+#[derive(Debug)]
+pub struct StorageChangeSet {
+	changes: Arc<Vec<(StorageKey, Option<StorageData>)>>,
+	child_changes: Arc<Vec<(StorageKey, Vec<(StorageKey, Option<StorageData>)>)>>,
+	filter: Option<HashSet<StorageKey>>,
+	child_filters: Option<HashMap<StorageKey, Option<HashSet<StorageKey>>>>,
+}
+
+impl StorageChangeSet {
+	/// Convert the change set into iterator over storage items.
+	pub fn iter<'a>(&'a self)
+		-> impl Iterator<Item=(Option<&'a StorageKey>, &'a StorageKey, Option<&'a StorageData>)> + 'a {
+		let top = self.changes
+			.iter()
+			.filter(move |&(key, _)| match self.filter {
+				Some(ref filter) => filter.contains(key),
+				None => true,
+			})
+			.map(move |(k,v)| (None, k, v.as_ref()));
+		let children = self.child_changes
+			.iter()
+			.filter_map(move |(sk, changes)| {
+				if let Some(cf) = self.child_filters.as_ref() {
+					if let Some(filter) = cf.get(sk) {
+						Some(changes
+							.iter()
+							.filter(move |&(key, _)| match filter {
+								Some(ref filter) => filter.contains(key),
+								None => true,
+							})
+							.map(move |(k,v)| (Some(sk), k, v.as_ref())))
+					} else { None }
+				} else { None	}
+			})
+			.flatten();
+		top.chain(children)
+	}
+}
+
+/// Type that implements `futures::Stream` of storage change events.
+pub type StorageEventStream<H> = mpsc::UnboundedReceiver<(H, StorageChangeSet)>;
+
+type SubscriberId = u64;
+
+/// Manages storage listeners.
+#[derive(Debug)]
+pub struct StorageNotifications<Block: BlockT> {
+	next_id: SubscriberId,
+	wildcard_listeners: FnvHashSet<SubscriberId>,
+	listeners: HashMap<StorageKey, FnvHashSet<SubscriberId>>,
+	child_listeners: HashMap<StorageKey, (
+		HashMap<StorageKey, FnvHashSet<SubscriberId>>,
+		FnvHashSet<SubscriberId>
+	)>,
+	sinks: FnvHashMap<SubscriberId, (
+		mpsc::UnboundedSender<(Block::Hash, StorageChangeSet)>,
+		Option<HashSet<StorageKey>>,
+		Option<HashMap<StorageKey, Option<HashSet<StorageKey>>>>,
+	)>,
+}
+
+impl<Block: BlockT> Default for StorageNotifications<Block> {
+	fn default() -> Self {
+		StorageNotifications {
+			next_id: Default::default(),
+			wildcard_listeners: Default::default(),
+			listeners: Default::default(),
+			child_listeners: Default::default(),
+			sinks: Default::default(),
+		}
+	}
+}
+
+impl<Block: BlockT> StorageNotifications<Block> {
+	/// Trigger notification to all listeners.
+	///
+	/// Note the changes are going to be filtered by listener's filter key.
+	/// In fact no event might be sent if clients are not interested in the changes.
+	pub fn trigger(
+		&mut self,
+		hash: &Block::Hash,
+		changeset: impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
+		child_changeset: impl Iterator<
+			Item=(Vec<u8>, impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)>)
+		>,
+	) {
+		let has_wildcard = !self.wildcard_listeners.is_empty();
+
+		// early exit if no listeners
+		if !has_wildcard && self.listeners.is_empty() && self.child_listeners.is_empty() {
+			return;
+		}
+
+		let mut subscribers = self.wildcard_listeners.clone();
+		let mut changes = Vec::new();
+		let mut child_changes = Vec::new();
+
+		// Collect subscribers and changes
+		for (k, v) in changeset {
+			let k = StorageKey(k);
+			let listeners = self.listeners.get(&k);
+
+			if let Some(ref listeners) = listeners {
+				subscribers.extend(listeners.iter());
+			}
+
+			if has_wildcard || listeners.is_some() {
+				changes.push((k, v.map(StorageData)));
+			}
+		}
+		for (sk, changeset) in child_changeset {
+			let sk = StorageKey(sk);
+			if let Some((cl, cw)) = self.child_listeners.get(&sk) {
+				let mut changes = Vec::new();
+				for (k, v) in changeset {
+					let k = StorageKey(k);
+					let listeners = cl.get(&k);
+
+					if let Some(ref listeners) = listeners {
+						subscribers.extend(listeners.iter());
+					}
+
+					subscribers.extend(cw.iter());
+
+					if !cw.is_empty() || listeners.is_some() {
+						changes.push((k, v.map(StorageData)));
+					}
+				}
+				if !changes.is_empty() {
+					child_changes.push((sk, changes));
+				}
+			}
+		}
+
+		// Don't send empty notifications
+		if changes.is_empty() && child_changes.is_empty() {
+			return;
+		}
+
+		let changes = Arc::new(changes);
+		let child_changes = Arc::new(child_changes);
+		// Trigger the events
+		for subscriber in subscribers {
+			let should_remove = {
+				let &(ref sink, ref filter, ref child_filters) = self.sinks.get(&subscriber)
+					.expect("subscribers returned from self.listeners are always in self.sinks; qed");
+				sink.unbounded_send((hash.clone(), StorageChangeSet {
+					changes: changes.clone(),
+					child_changes: child_changes.clone(),
+					filter: filter.clone(),
+					child_filters: child_filters.clone(),
+				})).is_err()
+			};
+
+			if should_remove {
+				self.remove_subscriber(subscriber);
+			}
+		}
+	}
+
+	fn remove_subscriber_from(
+		subscriber: &SubscriberId,
+		filters: &Option<HashSet<StorageKey>>,
+		listeners: &mut HashMap<StorageKey, FnvHashSet<SubscriberId>>,
+		wildcards: &mut FnvHashSet<SubscriberId>,
+	){
+		match filters {
+			None => {
+				wildcards.remove(subscriber);
+			},
+			Some(filters) => {
+
+				for key in filters.iter() {
+					let remove_key = match listeners.get_mut(key) {
+						Some(ref mut set) => {
+							set.remove(subscriber);
+							set.is_empty()
+						},
+						None => false,
+					};
+
+					if remove_key {
+						listeners.remove(key);
+					}
+				}
+			}
+		}
+	}
+
+	fn remove_subscriber(&mut self, subscriber: SubscriberId) {
+		if let Some((_, filters, child_filters)) = self.sinks.remove(&subscriber) {
+			Self::remove_subscriber_from(
+				&subscriber,
+				&filters,
+				&mut self.listeners,
+				&mut self.wildcard_listeners,
+			);
+			if let Some(child_filters) = child_filters.as_ref() {
+				for (c_key, filters) in child_filters {
+
+					if let Some((listeners, wildcards)) = self.child_listeners.get_mut(&c_key) {
+						Self::remove_subscriber_from(
+							&subscriber,
+							&filters,
+							&mut *listeners,
+							&mut *wildcards,
+						);
+
+						if listeners.is_empty() && wildcards.is_empty() {
+							self.child_listeners.remove(&c_key);
+						}
+					}
+				}
+			}
+		}
+	}
+
+	fn listen_from(
+		current_id: SubscriberId,
+		filter_keys: &Option<impl AsRef<[StorageKey]>>,
+		listeners: &mut HashMap<StorageKey, FnvHashSet<SubscriberId>>,
+		wildcards: &mut FnvHashSet<SubscriberId>,
+	) -> Option<HashSet<StorageKey>>
+	{
+		match filter_keys {
+			None => {
+				wildcards.insert(current_id);
+				None
+			},
+			Some(keys) => Some(keys.as_ref().iter().map(|key| {
+				listeners
+					.entry(key.clone())
+					.or_insert_with(Default::default)
+					.insert(current_id);
+				key.clone()
+			}).collect())
+		}
+	}
+
+	/// Start listening for particular storage keys.
+	pub fn listen(
+		&mut self,
+		filter_keys: Option<&[StorageKey]>,
+		filter_child_keys: Option<&[(StorageKey, Option<Vec<StorageKey>>)]>,
+	) -> StorageEventStream<Block::Hash> {
+		self.next_id += 1;
+		let current_id = self.next_id;
+
+		// add subscriber for every key
+		let keys = Self::listen_from(
+			current_id,
+			&filter_keys,
+			&mut self.listeners,
+			&mut self.wildcard_listeners,
+		);
+		let child_keys = filter_child_keys.map(|filter_child_keys| {
+			filter_child_keys.iter().map(|(c_key, o_keys)| {
+				let (c_listeners, c_wildcards) = self.child_listeners
+					.entry(c_key.clone())
+					.or_insert_with(Default::default);
+
+				(c_key.clone(), Self::listen_from(
+					current_id,
+					o_keys,
+					&mut *c_listeners,
+					&mut *c_wildcards,
+				))
+			}).collect()
+		});
+
+
+		// insert sink
+		let (tx, rx) = mpsc::unbounded();
+		self.sinks.insert(current_id, (tx, keys, child_keys));
+		rx
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use sr_primitives::testing::{H256 as Hash, Block as RawBlock, ExtrinsicWrapper};
+	use super::*;
+	use std::iter::{empty, Empty};
+
+	type TestChangeSet = (
+		Vec<(StorageKey, Option<StorageData>)>,
+		Vec<(StorageKey, Vec<(StorageKey, Option<StorageData>)>)>,
+	);
+
+	#[cfg(test)]
+	impl From<TestChangeSet> for StorageChangeSet {
+		fn from(changes: TestChangeSet) -> Self {
+			// warning hardcoded child trie wildcard to test upon
+			let child_filters = Some([
+				(StorageKey(vec![4]), None),
+				(StorageKey(vec![5]), None),
+			].into_iter().cloned().collect());
+			StorageChangeSet {
+				changes: Arc::new(changes.0),
+				child_changes: Arc::new(changes.1),
+				filter: None,
+				child_filters,
+			}
+		}
+	}
+
+	#[cfg(test)]
+	impl PartialEq for StorageChangeSet {
+		fn eq(&self, other: &Self) -> bool {
+			self.iter().eq(other.iter())
+		}
+	}
+
+	type Block = RawBlock<ExtrinsicWrapper<Hash>>;
+
+	#[test]
+	fn triggering_change_should_notify_wildcard_listeners() {
+		// given
+		let mut notifications = StorageNotifications::<Block>::default();
+		let child_filter = [(StorageKey(vec![4]), None)];
+		let mut recv = futures03::executor::block_on_stream(
+			notifications.listen(None, Some(&child_filter[..]))
+		);
+
+		// when
+		let changeset = vec![
+			(vec![2], Some(vec![3])),
+			(vec![3], None),
+		];
+		let c_changeset_1 = vec![
+			(vec![5], Some(vec![4])),
+			(vec![6], None),
+		];
+		let c_changeset = vec![(vec![4], c_changeset_1)];
+		notifications.trigger(
+			&Hash::from_low_u64_be(1),
+			changeset.into_iter(),
+			c_changeset.into_iter().map(|(a,b)| (a, b.into_iter())),
+		);
+
+		// then
+		assert_eq!(recv.next().unwrap(), (Hash::from_low_u64_be(1), (vec![
+			(StorageKey(vec![2]), Some(StorageData(vec![3]))),
+			(StorageKey(vec![3]), None),
+		], vec![(StorageKey(vec![4]), vec![
+			(StorageKey(vec![5]), Some(StorageData(vec![4]))),
+			(StorageKey(vec![6]), None),
+		])]).into()));
+	}
+
+	#[test]
+	fn should_only_notify_interested_listeners() {
+		// given
+		let mut notifications = StorageNotifications::<Block>::default();
+		let child_filter = [(StorageKey(vec![4]), Some(vec![StorageKey(vec![5])]))];
+		let mut recv1 = futures03::executor::block_on_stream(
+			notifications.listen(Some(&[StorageKey(vec![1])]), None)
+		);
+		let mut recv2 = futures03::executor::block_on_stream(
+			notifications.listen(Some(&[StorageKey(vec![2])]), None)
+		);
+		let mut recv3 = futures03::executor::block_on_stream(
+			notifications.listen(Some(&[]), Some(&child_filter))
+		);
+
+		// when
+		let changeset = vec![
+			(vec![2], Some(vec![3])),
+			(vec![1], None),
+		];
+		let c_changeset_1 = vec![
+			(vec![5], Some(vec![4])),
+			(vec![6], None),
+		];
+
+		let c_changeset = vec![(vec![4], c_changeset_1)];
+		notifications.trigger(
+			&Hash::from_low_u64_be(1),
+			changeset.into_iter(),
+			c_changeset.into_iter().map(|(a,b)| (a, b.into_iter())),
+		);
+
+		// then
+		assert_eq!(recv1.next().unwrap(), (Hash::from_low_u64_be(1), (vec![
+			(StorageKey(vec![1]), None),
+		], vec![]).into()));
+		assert_eq!(recv2.next().unwrap(), (Hash::from_low_u64_be(1), (vec![
+			(StorageKey(vec![2]), Some(StorageData(vec![3]))),
+		], vec![]).into()));
+		assert_eq!(recv3.next().unwrap(), (Hash::from_low_u64_be(1), (vec![],
+		vec![
+			(StorageKey(vec![4]), vec![(StorageKey(vec![5]), Some(StorageData(vec![4])))]),
+		]).into()));
+
+	}
+
+	#[test]
+	fn should_cleanup_subscribers_if_dropped() {
+		// given
+		let mut notifications = StorageNotifications::<Block>::default();
+		{
+			let child_filter = [(StorageKey(vec![4]), Some(vec![StorageKey(vec![5])]))];
+			let _recv1 = futures03::executor::block_on_stream(
+				notifications.listen(Some(&[StorageKey(vec![1])]), None)
+			);
+			let _recv2 = futures03::executor::block_on_stream(
+				notifications.listen(Some(&[StorageKey(vec![2])]), None)
+			);
+			let _recv3 = futures03::executor::block_on_stream(
+				notifications.listen(None, None)
+			);
+			let _recv4 = futures03::executor::block_on_stream(
+				notifications.listen(None, Some(&child_filter))
+			);
+			assert_eq!(notifications.listeners.len(), 2);
+			assert_eq!(notifications.wildcard_listeners.len(), 2);
+			assert_eq!(notifications.child_listeners.len(), 1);
+		}
+
+		// when
+		let changeset = vec![
+			(vec![2], Some(vec![3])),
+			(vec![1], None),
+		];
+		let c_changeset = empty::<(_, Empty<_>)>();
+		notifications.trigger(&Hash::from_low_u64_be(1), changeset.into_iter(), c_changeset);
+
+		// then
+		assert_eq!(notifications.listeners.len(), 0);
+		assert_eq!(notifications.wildcard_listeners.len(), 0);
+		assert_eq!(notifications.child_listeners.len(), 0);
+	}
+
+	#[test]
+	fn should_not_send_empty_notifications() {
+		// given
+		let mut recv = {
+			let mut notifications = StorageNotifications::<Block>::default();
+			let recv = futures03::executor::block_on_stream(notifications.listen(None, None));
+
+			// when
+			let changeset = vec![];
+			let c_changeset = empty::<(_, Empty<_>)>();
+			notifications.trigger(&Hash::from_low_u64_be(1), changeset.into_iter(), c_changeset);
+			recv
+		};
+
+		// then
+		assert_eq!(recv.next(), None);
+	}
+}

substrate/client/api/src/offchain.rs

@@ -0,0 +1,77 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::hash_map::{HashMap, Entry};
+
+/// Offchain workers local storage.
+pub trait OffchainStorage: Clone + Send + Sync {
+	/// Persist a value in storage under given key and prefix.
+	fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]);
+
+	/// Retrieve a value from storage under given key and prefix.
+	fn get(&self, prefix: &[u8], key: &[u8]) -> Option<Vec<u8>>;
+
+	/// Replace the value in storage if given old_value matches the current one.
+	///
+	/// Returns `true` if the value has been set and false otherwise.
+	fn compare_and_set(
+		&mut self,
+		prefix: &[u8],
+		key: &[u8],
+		old_value: Option<&[u8]>,
+		new_value: &[u8],
+	) -> bool;
+}
+
+/// In-memory storage for offchain workers.
+#[derive(Debug, Clone, Default)]
+pub struct InMemOffchainStorage {
+	storage: HashMap<Vec<u8>, Vec<u8>>,
+}
+
+impl OffchainStorage for InMemOffchainStorage {
+	fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) {
+		let key = prefix.iter().chain(key).cloned().collect();
+		self.storage.insert(key, value.to_vec());
+	}
+
+	fn get(&self, prefix: &[u8], key: &[u8]) -> Option<Vec<u8>> {
+		let key: Vec<u8> = prefix.iter().chain(key).cloned().collect();
+		self.storage.get(&key).cloned()
+	}
+
+	fn compare_and_set(
+		&mut self,
+		prefix: &[u8],
+		key: &[u8],
+		old_value: Option<&[u8]>,
+		new_value: &[u8],
+	) -> bool {
+		let key = prefix.iter().chain(key).cloned().collect();
+
+		match self.storage.entry(key) {
+			Entry::Vacant(entry) => if old_value.is_none() {
+				entry.insert(new_value.to_vec());
+				true
+			} else { false },
+			Entry::Occupied(ref mut entry) if Some(entry.get().as_slice()) == old_value => {
+				entry.insert(new_value.to_vec());
+				true
+			},
+			_ => false,
+		}
+	}
+}

substrate/client/authority-discovery/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "substrate-authority-discovery"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+build = "build.rs"
+
+[build-dependencies]
+prost-build = "0.5.0"
+
+[dependencies]
+authority-discovery-primitives = { package = "substrate-authority-discovery-primitives", path = "../../primitives/authority-discovery" }
+bytes = "0.4.12"
+client-api = { package = "substrate-client-api", path = "../api" }
+codec = { package = "parity-scale-codec", default-features = false, version = "1.0.3" }
+derive_more = "0.15.0"
+futures-preview = "0.3.0-alpha.19"
+futures-timer = "0.4"
+keystore = { package = "substrate-keystore", path = "../keystore" }
+libp2p = { version = "0.13.0", default-features = false, features = ["secp256k1", "libp2p-websocket"] }
+log = "0.4.8"
+network = { package = "substrate-network", path = "../network" }
+primitives = { package = "substrate-primitives", path = "../../primitives/core" }
+prost = "0.5.0"
+serde_json = "1.0.41"
+sr-primitives = { path = "../../primitives/sr-primitives" }
+
+[dev-dependencies]
+parking_lot = "0.9.0"
+peerset = { package = "substrate-peerset", path = "../../primitives/peerset" }
+test-client = { package = "substrate-test-runtime-client", path = "../../test/utils/runtime/client" }
+sr-api = { path = "../../primitives/sr-api" }

substrate/client/authority-discovery/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+	prost_build::compile_protos(&["src/schema/dht.proto"], &["src/schema"]).unwrap();
+}

substrate/client/authority-discovery/src/error.rs

@@ -0,0 +1,45 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Authority discovery errors.
+
+/// AuthorityDiscovery Result.
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Error type for the authority discovery module.
+#[derive(Debug, derive_more::Display, derive_more::From)]
+pub enum Error {
+	/// Failed to verify a dht payload with the given signature.
+	VerifyingDhtPayload,
+	/// Failed to hash the authority id to be used as a dht key.
+	HashingAuthorityId(libp2p::core::multiaddr::multihash::EncodeError),
+	/// Failed calling into the Substrate runtime.
+	CallingRuntime(client_api::error::Error),
+	/// From the Dht we only get the hashed authority id. In order to retrieve the actual authority id and to ensure it
+	/// is actually an authority, we match the hash against the hash of the authority id of all other authorities. This
+	/// error is the result of the above failing.
+	MatchingHashedAuthorityIdWithAuthorityId,
+	/// Failed to set the authority discovery peerset priority group in the peerset module.
+	SettingPeersetPriorityGroup(String),
+	/// Failed to encode a protobuf payload.
+	EncodingProto(prost::EncodeError),
+	/// Failed to decode a protobuf payload.
+	DecodingProto(prost::DecodeError),
+	/// Failed to encode or decode scale payload
+	EncodingDecodingScale(codec::Error),
+	/// Failed to parse a libp2p multi address.
+	ParsingMultiaddress(libp2p::core::multiaddr::Error),
+}

substrate/client/authority-discovery/src/lib.rs

@@ -0,0 +1,731 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+#![warn(missing_docs)]
+
+//! Substrate authority discovery.
+//!
+//! This crate enables Substrate authorities to directly connect to other authorities. [`AuthorityDiscovery`] implements
+//! the Future trait. By polling [`AuthorityDiscovery`] an authority:
+//!
+//!
+//! 1. **Makes itself discoverable**
+//!
+//!    1. Retrieves its external addresses.
+//!
+//!    2. Adds its network peer id to the addresses.
+//!
+//!    3. Signs the above.
+//!
+//!    4. Puts the signature and the addresses on the libp2p Kademlia DHT.
+//!
+//!
+//! 2. **Discovers other authorities**
+//!
+//!    1. Retrieves the current set of authorities.
+//!
+//!    2. Starts DHT queries for the ids of the authorities.
+//!
+//!    3. Validates the signatures of the retrieved key value pairs.
+//!
+//!    4. Adds the retrieved external addresses as priority nodes to the peerset.
+use std::collections::{HashMap, HashSet};
+use std::convert::TryInto;
+use std::iter::FromIterator;
+use std::marker::PhantomData;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use futures::task::{Context, Poll};
+use futures::Future;
+use futures_timer::Interval;
+use futures::prelude::*;
+
+use authority_discovery_primitives::{AuthorityDiscoveryApi, AuthorityId, AuthoritySignature, AuthorityPair};
+use client_api::blockchain::HeaderBackend;
+use codec::{Decode, Encode};
+use error::{Error, Result};
+use log::{debug, error, log_enabled, warn};
+use network::specialization::NetworkSpecialization;
+use network::{DhtEvent, ExHashT};
+use primitives::crypto::{key_types, Pair};
+use primitives::traits::BareCryptoStorePtr;
+use prost::Message;
+use sr_primitives::generic::BlockId;
+use sr_primitives::traits::{Block as BlockT, ProvideRuntimeApi};
+
+mod error;
+/// Dht payload schemas generated from Protobuf definitions via Prost crate in build.rs.
+mod schema {
+	include!(concat!(env!("OUT_DIR"), "/authority_discovery.rs"));
+}
+
+/// Upper bound estimation on how long one should wait before accessing the Kademlia DHT.
+const LIBP2P_KADEMLIA_BOOTSTRAP_TIME: Duration = Duration::from_secs(30);
+
+/// Name of the Substrate peerset priority group for authorities discovered through the authority discovery module.
+const AUTHORITIES_PRIORITY_GROUP_NAME: &'static str = "authorities";
+
+/// An `AuthorityDiscovery` makes a given authority discoverable and discovers other authorities.
+pub struct AuthorityDiscovery<Client, Network, Block>
+where
+	Block: BlockT + 'static,
+	Network: NetworkProvider,
+	Client: ProvideRuntimeApi + Send + Sync + 'static + HeaderBackend<Block>,
+	<Client as ProvideRuntimeApi>::Api: AuthorityDiscoveryApi<Block>,
+
+{
+	client: Arc<Client>,
+
+	network: Arc<Network>,
+	/// Channel we receive Dht events on.
+	dht_event_rx: Pin<Box<dyn Stream<Item = DhtEvent> + Send>>,
+
+	key_store: BareCryptoStorePtr,
+
+	/// Interval to be proactive, publishing own addresses.
+	publish_interval: Interval,
+	/// Interval on which to query for addresses of other authorities.
+	query_interval: Interval,
+
+	/// The network peerset interface for priority groups lets us only set an entire group, but we retrieve the
+	/// addresses of other authorities one by one from the network. To use the peerset interface we need to cache the
+	/// addresses and always overwrite the entire peerset priority group. To ensure this map doesn't grow indefinitely
+	/// `purge_old_authorities_from_cache` function is called each time we add a new entry.
+	address_cache: HashMap<AuthorityId, Vec<libp2p::Multiaddr>>,
+
+	phantom: PhantomData<Block>,
+}
+
+impl<Client, Network, Block> AuthorityDiscovery<Client, Network, Block>
+where
+	Block: BlockT + Unpin + 'static,
+	Network: NetworkProvider,
+	Client: ProvideRuntimeApi + Send + Sync + 'static + HeaderBackend<Block>,
+	<Client as ProvideRuntimeApi>::Api: AuthorityDiscoveryApi<Block, Error = client_api::error::Error>,
+	Self: Future<Output = ()>,
+{
+	/// Return a new authority discovery.
+	pub fn new(
+		client: Arc<Client>,
+		network: Arc<Network>,
+		key_store: BareCryptoStorePtr,
+		dht_event_rx: Pin<Box<dyn Stream<Item = DhtEvent> + Send>>,
+	) -> Self {
+		// Kademlia's default time-to-live for Dht records is 36h, republishing records every 24h. Given that a node
+		// could restart at any point in time, one can not depend on the republishing process, thus publishing own
+		// external addresses should happen on an interval < 36h.
+		let publish_interval = Interval::new_at(
+			Instant::now() + LIBP2P_KADEMLIA_BOOTSTRAP_TIME,
+			Duration::from_secs(12 * 60 * 60),
+		);
+
+		// External addresses of other authorities can change at any given point in time. The interval on which to query
+		// for external addresses of other authorities is a trade off between efficiency and performance.
+		let query_interval = Interval::new_at(
+			Instant::now() + LIBP2P_KADEMLIA_BOOTSTRAP_TIME,
+			Duration::from_secs(10 * 60),
+		);
+
+		let address_cache = HashMap::new();
+
+		AuthorityDiscovery {
+			client,
+			network,
+			dht_event_rx,
+			key_store,
+			publish_interval,
+			query_interval,
+			address_cache,
+			phantom: PhantomData,
+		}
+	}
+
+	fn publish_own_ext_addresses(&mut self) -> Result<()> {
+		let addresses = self
+			.network
+			.external_addresses()
+			.into_iter()
+			.map(|a| {
+				a.with(libp2p::core::multiaddr::Protocol::P2p(
+					self.network.local_peer_id().into(),
+				))
+			})
+			.map(|a| a.to_vec())
+			.collect();
+
+		let mut serialized_addresses = vec![];
+		schema::AuthorityAddresses { addresses }
+			.encode(&mut serialized_addresses)
+			.map_err(Error::EncodingProto)?;
+
+		for key in self.get_priv_keys_within_authority_set()?.into_iter() {
+			let signature = key.sign(&serialized_addresses);
+
+			let mut signed_addresses = vec![];
+			schema::SignedAuthorityAddresses {
+				addresses: serialized_addresses.clone(),
+				signature: signature.encode(),
+			}
+			.encode(&mut signed_addresses)
+				.map_err(Error::EncodingProto)?;
+
+			self.network.put_value(
+				hash_authority_id(key.public().as_ref())?,
+				signed_addresses,
+			);
+		}
+
+		Ok(())
+	}
+
+	fn request_addresses_of_others(&mut self) -> Result<()> {
+		let id = BlockId::hash(self.client.info().best_hash);
+
+		let authorities = self
+			.client
+			.runtime_api()
+			.authorities(&id)
+			.map_err(Error::CallingRuntime)?;
+
+		for authority_id in authorities.iter() {
+			self.network
+				.get_value(&hash_authority_id(authority_id.as_ref())?);
+		}
+
+		Ok(())
+	}
+
+	fn handle_dht_events(&mut self, cx: &mut Context) -> Result<()> {
+		while let Poll::Ready(Some(event)) = self.dht_event_rx.poll_next_unpin(cx) {
+			match event {
+				DhtEvent::ValueFound(v) => {
+					if log_enabled!(log::Level::Debug) {
+						let hashes = v.iter().map(|(hash, _value)| hash.clone());
+						debug!(target: "sub-authority-discovery", "Value for hash '{:?}' found on Dht.", hashes);
+					}
+
+					self.handle_dht_value_found_event(v)?;
+				}
+				DhtEvent::ValueNotFound(hash) => warn!(
+					target: "sub-authority-discovery",
+					"Value for hash '{:?}' not found on Dht.", hash
+				),
+				DhtEvent::ValuePut(hash) => debug!(
+					target: "sub-authority-discovery",
+					"Successfully put hash '{:?}' on Dht.", hash),
+				DhtEvent::ValuePutFailed(hash) => warn!(
+					target: "sub-authority-discovery",
+					"Failed to put hash '{:?}' on Dht.", hash
+				),
+			}
+		}
+
+		Ok(())
+	}
+
+	fn handle_dht_value_found_event(
+		&mut self,
+		values: Vec<(libp2p::kad::record::Key, Vec<u8>)>,
+	) -> Result<()> {
+		debug!(target: "sub-authority-discovery", "Got Dht value from network.");
+
+		let block_id = BlockId::hash(self.client.info().best_hash);
+
+		// From the Dht we only get the hashed authority id. In order to retrieve the actual authority id and to ensure
+		// it is actually an authority, we match the hash against the hash of the authority id of all other authorities.
+		let authorities = self.client.runtime_api().authorities(&block_id)?;
+		self.purge_old_authorities_from_cache(&authorities);
+
+		let authorities = authorities
+			.into_iter()
+			.map(|id| hash_authority_id(id.as_ref()).map(|h| (h, id)))
+			.collect::<Result<HashMap<_, _>>>()?;
+
+		for (key, value) in values.iter() {
+			// Check if the event origins from an authority in the current authority set.
+			let authority_id: &AuthorityId = authorities
+				.get(key)
+				.ok_or(Error::MatchingHashedAuthorityIdWithAuthorityId)?;
+
+			let schema::SignedAuthorityAddresses {
+				signature,
+				addresses,
+			} = schema::SignedAuthorityAddresses::decode(value).map_err(Error::DecodingProto)?;
+			let signature = AuthoritySignature::decode(&mut &signature[..]).map_err(Error::EncodingDecodingScale)?;
+
+			if !AuthorityPair::verify(&signature, &addresses, authority_id) {
+				return Err(Error::VerifyingDhtPayload);
+			}
+
+			let addresses: Vec<libp2p::Multiaddr> = schema::AuthorityAddresses::decode(addresses)
+				.map(|a| a.addresses)
+				.map_err(Error::DecodingProto)?
+				.into_iter()
+				.map(|a| a.try_into())
+				.collect::<std::result::Result<_, _>>()
+				.map_err(Error::ParsingMultiaddress)?;
+
+			self.address_cache.insert(authority_id.clone(), addresses);
+		}
+
+		// Let's update the peerset priority group with all the addresses we have in our cache.
+
+		let addresses = HashSet::from_iter(
+			self.address_cache
+				.iter()
+				.map(|(_peer_id, addresses)| addresses.clone())
+				.flatten(),
+		);
+
+		debug!(target: "sub-authority-discovery", "Applying priority group {:#?} to peerset.", addresses);
+		self.network
+			.set_priority_group(AUTHORITIES_PRIORITY_GROUP_NAME.to_string(), addresses)
+			.map_err(Error::SettingPeersetPriorityGroup)?;
+
+		Ok(())
+	}
+
+	fn purge_old_authorities_from_cache(&mut self, current_authorities: &Vec<AuthorityId>) {
+		self.address_cache
+			.retain(|peer_id, _addresses| current_authorities.contains(peer_id))
+	}
+
+	/// Retrieve all local authority discovery private keys that are within the current authority
+	/// set.
+	fn get_priv_keys_within_authority_set(&mut self) -> Result<Vec<AuthorityPair>> {
+		let keys = self.get_own_public_keys_within_authority_set()?
+			.into_iter()
+			.map(std::convert::Into::into)
+			.filter_map(|pub_key| {
+				self.key_store.read().sr25519_key_pair(key_types::AUTHORITY_DISCOVERY, &pub_key)
+			})
+			.map(std::convert::Into::into)
+			.collect();
+
+		Ok(keys)
+	}
+
+	/// Retrieve our public keys within the current authority set.
+	//
+	// A node might have multiple authority discovery keys within its keystore, e.g. an old one and
+	// one for the upcoming session. In addition it could be participating in the current authority
+	// set with two keys. The function does not return all of the local authority discovery public
+	// keys, but only the ones intersecting with the current authority set.
+	fn get_own_public_keys_within_authority_set(&mut self) -> Result<HashSet<AuthorityId>> {
+		let local_pub_keys = self.key_store
+			.read()
+			.sr25519_public_keys(key_types::AUTHORITY_DISCOVERY)
+			.into_iter()
+			.collect::<HashSet<_>>();
+
+		let id = BlockId::hash(self.client.info().best_hash);
+		let current_authorities = self
+			.client
+			.runtime_api()
+			.authorities(&id)
+			.map_err(Error::CallingRuntime)?
+			.into_iter()
+			.map(std::convert::Into::into)
+			.collect::<HashSet<_>>();
+
+		let intersection = local_pub_keys.intersection(&current_authorities)
+			.cloned()
+			.map(std::convert::Into::into)
+			.collect();
+
+		Ok(intersection)
+	}
+}
+
+impl<Client, Network, Block> Future for AuthorityDiscovery<Client, Network, Block>
+where
+	Block: BlockT + Unpin + 'static,
+	Network: NetworkProvider,
+	Client: ProvideRuntimeApi + Send + Sync + 'static + HeaderBackend<Block>,
+	<Client as ProvideRuntimeApi>::Api: AuthorityDiscoveryApi<Block, Error = client_api::error::Error>,
+{
+	type Output = ();
+
+	fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Self::Output> {
+		let mut inner = || -> Result<()> {
+			// Process incoming events before triggering new ones.
+			self.handle_dht_events(cx)?;
+
+			if let Poll::Ready(_) = self.publish_interval.poll_next_unpin(cx) {
+				// Make sure to call interval.poll until it returns Async::NotReady once. Otherwise, in case one of the
+				// function calls within this block do a `return`, we don't call `interval.poll` again and thereby the
+				// underlying Tokio task is never registered with Tokio's Reactor to be woken up on the next interval
+				// tick.
+				while let Poll::Ready(_) = self.publish_interval.poll_next_unpin(cx) {}
+
+				self.publish_own_ext_addresses()?;
+			}
+
+			if let Poll::Ready(_) = self.query_interval.poll_next_unpin(cx) {
+				// Make sure to call interval.poll until it returns Async::NotReady once. Otherwise, in case one of the
+				// function calls within this block do a `return`, we don't call `interval.poll` again and thereby the
+				// underlying Tokio task is never registered with Tokio's Reactor to be woken up on the next interval
+				// tick.
+				while let Poll::Ready(_) = self.query_interval.poll_next_unpin(cx) {}
+
+				self.request_addresses_of_others()?;
+			}
+
+			Ok(())
+		};
+
+		match inner() {
+			Ok(()) => {}
+			Err(e) => error!(target: "sub-authority-discovery", "Poll failure: {:?}", e),
+		};
+
+		// Make sure to always return NotReady as this is a long running task with the same lifetime as the node itself.
+		Poll::Pending
+	}
+}
+
+/// NetworkProvider provides AuthorityDiscovery with all necessary hooks into the underlying Substrate networking. Using
+/// this trait abstraction instead of NetworkService directly is necessary to unit test AuthorityDiscovery.
+pub trait NetworkProvider {
+	/// Returns the local external addresses.
+	fn external_addresses(&self) -> Vec<libp2p::Multiaddr>;
+
+	/// Returns the network identity of the node.
+	fn local_peer_id(&self) -> libp2p::PeerId;
+
+	/// Modify a peerset priority group.
+	fn set_priority_group(
+		&self,
+		group_id: String,
+		peers: HashSet<libp2p::Multiaddr>,
+	) -> std::result::Result<(), String>;
+
+	/// Start putting a value in the Dht.
+	fn put_value(&self, key: libp2p::kad::record::Key, value: Vec<u8>);
+
+	/// Start getting a value from the Dht.
+	fn get_value(&self, key: &libp2p::kad::record::Key);
+}
+
+impl<B, S, H> NetworkProvider for network::NetworkService<B, S, H>
+where
+	B: BlockT + 'static,
+	S: NetworkSpecialization<B>,
+	H: ExHashT,
+{
+	fn external_addresses(&self) -> Vec<libp2p::Multiaddr> {
+		self.external_addresses()
+	}
+	fn local_peer_id(&self) -> libp2p::PeerId {
+		self.local_peer_id()
+	}
+	fn set_priority_group(
+		&self,
+		group_id: String,
+		peers: HashSet<libp2p::Multiaddr>,
+	) -> std::result::Result<(), String> {
+		self.set_priority_group(group_id, peers)
+	}
+	fn put_value(&self, key: libp2p::kad::record::Key, value: Vec<u8>) {
+		self.put_value(key, value)
+	}
+	fn get_value(&self, key: &libp2p::kad::record::Key) {
+		self.get_value(key)
+	}
+}
+
+fn hash_authority_id(id: &[u8]) -> Result<libp2p::kad::record::Key> {
+	libp2p::multihash::encode(libp2p::multihash::Hash::SHA2256, id)
+		.map(|k| libp2p::kad::record::Key::new(&k))
+		.map_err(Error::HashingAuthorityId)
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use sr_api::{ApiExt, Core, RuntimeVersion, StorageProof};
+	use futures::channel::mpsc::channel;
+	use futures::executor::block_on;
+	use futures::future::poll_fn;
+	use primitives::{ExecutionContext, NativeOrEncoded, testing::KeyStore};
+	use sr_primitives::traits::Zero;
+	use sr_primitives::traits::{ApiRef, Block as BlockT, NumberFor, ProvideRuntimeApi};
+	use std::sync::{Arc, Mutex};
+	use test_client::runtime::Block;
+
+	#[derive(Clone)]
+	struct TestApi {
+		authorities: Vec<AuthorityId>,
+	}
+
+	impl ProvideRuntimeApi for TestApi {
+		type Api = RuntimeApi;
+
+		fn runtime_api<'a>(&'a self) -> ApiRef<'a, Self::Api> {
+			RuntimeApi{authorities: self.authorities.clone()}.into()
+		}
+	}
+
+	/// Blockchain database header backend. Does not perform any validation.
+	impl<Block: BlockT> HeaderBackend<Block> for TestApi {
+		fn header(
+			&self,
+			_id: BlockId<Block>,
+		) -> std::result::Result<Option<Block::Header>, client_api::error::Error> {
+			Ok(None)
+		}
+
+		fn info(&self) -> client_api::blockchain::Info<Block> {
+			client_api::blockchain::Info {
+				best_hash: Default::default(),
+				best_number: Zero::zero(),
+				finalized_hash: Default::default(),
+				finalized_number: Zero::zero(),
+				genesis_hash: Default::default(),
+			}
+		}
+
+		fn status(
+			&self,
+			_id: BlockId<Block>,
+		) -> std::result::Result<client_api::blockchain::BlockStatus, client_api::error::Error> {
+			Ok(client_api::blockchain::BlockStatus::Unknown)
+		}
+
+		fn number(
+			&self,
+			_hash: Block::Hash,
+		) -> std::result::Result<Option<NumberFor<Block>>, client_api::error::Error> {
+			Ok(None)
+		}
+
+		fn hash(
+			&self,
+			_number: NumberFor<Block>,
+		) -> std::result::Result<Option<Block::Hash>, client_api::error::Error> {
+			Ok(None)
+		}
+	}
+
+	struct RuntimeApi {
+		authorities: Vec<AuthorityId>,
+	}
+
+	impl Core<Block> for RuntimeApi {
+		fn Core_version_runtime_api_impl(
+			&self,
+			_: &BlockId<Block>,
+			_: ExecutionContext,
+			_: Option<()>,
+			_: Vec<u8>,
+		) -> std::result::Result<NativeOrEncoded<RuntimeVersion>, client_api::error::Error> {
+			unimplemented!("Not required for testing!")
+		}
+
+		fn Core_execute_block_runtime_api_impl(
+			&self,
+			_: &BlockId<Block>,
+			_: ExecutionContext,
+			_: Option<(Block)>,
+			_: Vec<u8>,
+		) -> std::result::Result<NativeOrEncoded<()>, client_api::error::Error> {
+			unimplemented!("Not required for testing!")
+		}
+
+		fn Core_initialize_block_runtime_api_impl(
+			&self,
+			_: &BlockId<Block>,
+			_: ExecutionContext,
+			_: Option<&<Block as BlockT>::Header>,
+			_: Vec<u8>,
+		) -> std::result::Result<NativeOrEncoded<()>, client_api::error::Error> {
+			unimplemented!("Not required for testing!")
+		}
+	}
+
+	impl ApiExt<Block> for RuntimeApi {
+		type Error = client_api::error::Error;
+
+		fn map_api_result<F: FnOnce(&Self) -> std::result::Result<R, E>, R, E>(
+			&self,
+			_: F,
+		) -> std::result::Result<R, E> {
+			unimplemented!("Not required for testing!")
+		}
+
+		fn runtime_version_at(
+			&self,
+			_: &BlockId<Block>,
+		) -> std::result::Result<RuntimeVersion, client_api::error::Error> {
+			unimplemented!("Not required for testing!")
+		}
+
+		fn record_proof(&mut self) {
+			unimplemented!("Not required for testing!")
+		}
+
+		fn extract_proof(&mut self) -> Option<StorageProof> {
+			unimplemented!("Not required for testing!")
+		}
+	}
+
+	impl AuthorityDiscoveryApi<Block> for RuntimeApi {
+		fn AuthorityDiscoveryApi_authorities_runtime_api_impl(
+			&self,
+			_: &BlockId<Block>,
+			_: ExecutionContext,
+			_: Option<()>,
+			_: Vec<u8>,
+		) -> std::result::Result<NativeOrEncoded<Vec<AuthorityId>>, client_api::error::Error> {
+			return Ok(NativeOrEncoded::Native(self.authorities.clone()));
+		}
+	}
+
+	#[derive(Default)]
+	struct TestNetwork {
+		// Whenever functions on `TestNetwork` are called, the function arguments are added to the vectors below.
+		pub put_value_call: Arc<Mutex<Vec<(libp2p::kad::record::Key, Vec<u8>)>>>,
+		pub get_value_call: Arc<Mutex<Vec<libp2p::kad::record::Key>>>,
+		pub set_priority_group_call: Arc<Mutex<Vec<(String, HashSet<libp2p::Multiaddr>)>>>,
+	}
+
+	impl NetworkProvider for TestNetwork {
+		fn external_addresses(&self) -> Vec<libp2p::Multiaddr> {
+			vec![]
+		}
+		fn local_peer_id(&self) -> libp2p::PeerId {
+			libp2p::PeerId::random()
+		}
+		fn set_priority_group(
+			&self,
+			group_id: String,
+			peers: HashSet<libp2p::Multiaddr>,
+		) -> std::result::Result<(), String> {
+			self.set_priority_group_call
+				.lock()
+				.unwrap()
+				.push((group_id, peers));
+			Ok(())
+		}
+		fn put_value(&self, key: libp2p::kad::record::Key, value: Vec<u8>) {
+			self.put_value_call.lock().unwrap().push((key, value));
+		}
+		fn get_value(&self, key: &libp2p::kad::record::Key) {
+			self.get_value_call.lock().unwrap().push(key.clone());
+		}
+	}
+
+	#[test]
+	fn publish_own_ext_addresses_puts_record_on_dht() {
+		let (_dht_event_tx, dht_event_rx) = channel(1000);
+		let network: Arc<TestNetwork> = Arc::new(Default::default());
+		let key_store = KeyStore::new();
+		let public = key_store.write().sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None).unwrap();
+		let test_api = Arc::new(TestApi {authorities: vec![public.into()]});
+
+		let mut authority_discovery =
+			AuthorityDiscovery::new(test_api, network.clone(), key_store, dht_event_rx.boxed());
+
+		authority_discovery.publish_own_ext_addresses().unwrap();
+
+		// Expect authority discovery to put a new record onto the dht.
+		assert_eq!(network.put_value_call.lock().unwrap().len(), 1);
+	}
+
+	#[test]
+	fn request_addresses_of_others_triggers_dht_get_query() {
+		let (_dht_event_tx, dht_event_rx) = channel(1000);
+
+		// Generate authority keys
+		let authority_1_key_pair = AuthorityPair::from_seed_slice(&[1; 32]).unwrap();
+		let authority_2_key_pair = AuthorityPair::from_seed_slice(&[2; 32]).unwrap();
+
+		let test_api = Arc::new(TestApi {
+			authorities: vec![authority_1_key_pair.public(), authority_2_key_pair.public()],
+		});
+
+		let network: Arc<TestNetwork> = Arc::new(Default::default());
+		let key_store = KeyStore::new();
+
+		let mut authority_discovery =
+			AuthorityDiscovery::new(test_api, network.clone(), key_store, dht_event_rx.boxed());
+
+		authority_discovery.request_addresses_of_others().unwrap();
+
+		// Expect authority discovery to request new records from the dht.
+		assert_eq!(network.get_value_call.lock().unwrap().len(), 2);
+	}
+
+	#[test]
+	fn handle_dht_events_with_value_found_should_call_set_priority_group() {
+		// Create authority discovery.
+
+		let (mut dht_event_tx, dht_event_rx) = channel(1000);
+		let key_pair = AuthorityPair::from_seed_slice(&[1; 32]).unwrap();
+		let test_api = Arc::new(TestApi {authorities: vec![key_pair.public()]});
+		let network: Arc<TestNetwork> = Arc::new(Default::default());
+		let key_store = KeyStore::new();
+
+		let mut authority_discovery =
+			AuthorityDiscovery::new(test_api, network.clone(), key_store, dht_event_rx.boxed());
+
+		// Create sample dht event.
+
+		let authority_id_1 = hash_authority_id(key_pair.public().as_ref()).unwrap();
+		let address_1: libp2p::Multiaddr = "/ip6/2001:db8::".parse().unwrap();
+
+		let mut serialized_addresses = vec![];
+		schema::AuthorityAddresses {
+			addresses: vec![address_1.to_vec()],
+		}
+		.encode(&mut serialized_addresses)
+		.unwrap();
+
+		let signature = key_pair.sign(serialized_addresses.as_ref()).encode();
+		let mut signed_addresses = vec![];
+		schema::SignedAuthorityAddresses {
+			addresses: serialized_addresses,
+			signature: signature,
+		}
+		.encode(&mut signed_addresses)
+		.unwrap();
+
+		let dht_event = network::DhtEvent::ValueFound(vec![(authority_id_1, signed_addresses)]);
+		dht_event_tx.try_send(dht_event).unwrap();
+
+		// Make authority discovery handle the event.
+		let f = |cx: &mut Context<'_>| -> Poll<()> {
+			authority_discovery.handle_dht_events(cx).unwrap();
+
+			// Expect authority discovery to set the priority set.
+			assert_eq!(network.set_priority_group_call.lock().unwrap().len(), 1);
+
+			assert_eq!(
+				network.set_priority_group_call.lock().unwrap()[0],
+				(
+					"authorities".to_string(),
+					HashSet::from_iter(vec![address_1.clone()].into_iter())
+				)
+			);
+
+			Poll::Ready(())
+		};
+
+		let _ = block_on(poll_fn(f));
+	}
+}

substrate/client/authority-discovery/src/schema/dht.proto

@@ -0,0 +1,14 @@
+syntax = "proto3";
+
+package authority_discovery;
+
+// First we need to serialize the addresses in order to be able to sign them.
+message AuthorityAddresses {
+	repeated bytes addresses = 1;
+}
+
+// Then we need to serialize addresses and signature to send them over the wire.
+message SignedAuthorityAddresses {
+	bytes addresses = 1;
+	bytes signature = 2;
+}

substrate/client/basic-authorship/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "substrate-basic-authorship"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+log = "0.4.8"
+futures-preview = "0.3.0-alpha.19"
+codec = { package = "parity-scale-codec", version = "1.0.0" }
+sr-primitives = { path = "../../primitives/sr-primitives" }
+primitives = { package = "substrate-primitives", path = "../../primitives/core" }
+client = { package = "substrate-client", path = "../" }
+client-api = { package = "substrate-client-api", path = "../api" }
+consensus_common = { package = "substrate-consensus-common", path = "../../primitives/consensus/common" }
+inherents = { package = "substrate-inherents", path = "../../primitives/inherents" }
+substrate-telemetry = { path = "../telemetry" }
+transaction_pool = { package = "substrate-transaction-pool", path = "../transaction-pool" }
+block-builder = { package = "substrate-block-builder", path = "../block-builder" }
+
+[dev-dependencies]
+test-client = { package = "substrate-test-runtime-client", path = "../../test/utils/runtime/client" }

substrate/client/basic-authorship/src/basic_authorship.rs

@@ -0,0 +1,283 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! A consensus proposer for "basic" chains which use the primitive inherent-data.
+
+// FIXME #1021 move this into substrate-consensus-common
+//
+
+use std::{time, sync::Arc};
+use client_api::{error, CallExecutor};
+use client::Client as SubstrateClient;
+use codec::Decode;
+use consensus_common::{evaluation};
+use inherents::InherentData;
+use log::{error, info, debug, trace};
+use primitives::{H256, Blake2Hasher, ExecutionContext};
+use sr_primitives::{
+	traits::{
+		Block as BlockT, Hash as HashT, Header as HeaderT, ProvideRuntimeApi, DigestFor, BlakeTwo256
+	},
+	generic::BlockId,
+};
+use transaction_pool::txpool::{self, Pool as TransactionPool};
+use substrate_telemetry::{telemetry, CONSENSUS_INFO};
+use block_builder::BlockBuilderApi;
+
+/// Proposer factory.
+pub struct ProposerFactory<C, A> where A: txpool::ChainApi {
+	/// The client instance.
+	pub client: Arc<C>,
+	/// The transaction pool.
+	pub transaction_pool: Arc<TransactionPool<A>>,
+}
+
+impl<B, E, Block, RA, A> consensus_common::Environment<Block> for
+ProposerFactory<SubstrateClient<B, E, Block, RA>, A>
+where
+	A: txpool::ChainApi<Block=Block>,
+	B: client_api::backend::Backend<Block, Blake2Hasher> + Send + Sync + 'static,
+	E: CallExecutor<Block, Blake2Hasher> + Send + Sync + Clone + 'static,
+	Block: BlockT<Hash=H256>,
+	RA: Send + Sync + 'static,
+	SubstrateClient<B, E, Block, RA>: ProvideRuntimeApi,
+	<SubstrateClient<B, E, Block, RA> as ProvideRuntimeApi>::Api:
+		BlockBuilderApi<Block, Error = error::Error>,
+{
+	type Proposer = Proposer<Block, SubstrateClient<B, E, Block, RA>, A>;
+	type Error = error::Error;
+
+	fn init(
+		&mut self,
+		parent_header: &<Block as BlockT>::Header,
+	) -> Result<Self::Proposer, error::Error> {
+		let parent_hash = parent_header.hash();
+
+		let id = BlockId::hash(parent_hash);
+
+		info!("Starting consensus session on top of parent {:?}", parent_hash);
+
+		let proposer = Proposer {
+			client: self.client.clone(),
+			parent_hash,
+			parent_id: id,
+			parent_number: *parent_header.number(),
+			transaction_pool: self.transaction_pool.clone(),
+			now: Box::new(time::Instant::now),
+		};
+
+		Ok(proposer)
+	}
+}
+
+/// The proposer logic.
+pub struct Proposer<Block: BlockT, C, A: txpool::ChainApi> {
+	client: Arc<C>,
+	parent_hash: <Block as BlockT>::Hash,
+	parent_id: BlockId<Block>,
+	parent_number: <<Block as BlockT>::Header as HeaderT>::Number,
+	transaction_pool: Arc<TransactionPool<A>>,
+	now: Box<dyn Fn() -> time::Instant>,
+}
+
+impl<B, E, Block, RA, A> consensus_common::Proposer<Block> for
+Proposer<Block, SubstrateClient<B, E, Block, RA>, A>
+where
+	A: txpool::ChainApi<Block=Block>,
+	B: client_api::backend::Backend<Block, Blake2Hasher> + Send + Sync + 'static,
+	E: CallExecutor<Block, Blake2Hasher> + Send + Sync + Clone + 'static,
+	Block: BlockT<Hash=H256>,
+	RA: Send + Sync + 'static,
+	SubstrateClient<B, E, Block, RA>: ProvideRuntimeApi,
+	<SubstrateClient<B, E, Block, RA> as ProvideRuntimeApi>::Api:
+		BlockBuilderApi<Block, Error = error::Error>,
+{
+	type Create = futures::future::Ready<Result<Block, error::Error>>;
+	type Error = error::Error;
+
+	fn propose(
+		&mut self,
+		inherent_data: InherentData,
+		inherent_digests: DigestFor<Block>,
+		max_duration: time::Duration,
+	) -> Self::Create {
+		// leave some time for evaluation and block finalization (33%)
+		let deadline = (self.now)() + max_duration - max_duration / 3;
+		futures::future::ready(self.propose_with(inherent_data, inherent_digests, deadline))
+	}
+}
+
+impl<Block, B, E, RA, A> Proposer<Block, SubstrateClient<B, E, Block, RA>, A>	where
+	A: txpool::ChainApi<Block=Block>,
+	B: client_api::backend::Backend<Block, Blake2Hasher> + Send + Sync + 'static,
+	E: CallExecutor<Block, Blake2Hasher> + Send + Sync + Clone + 'static,
+	Block: BlockT<Hash=H256>,
+	RA: Send + Sync + 'static,
+	SubstrateClient<B, E, Block, RA>: ProvideRuntimeApi,
+	<SubstrateClient<B, E, Block, RA> as ProvideRuntimeApi>::Api:
+		BlockBuilderApi<Block, Error = error::Error>,
+{
+	fn propose_with(
+		&self,
+		inherent_data: InherentData,
+		inherent_digests: DigestFor<Block>,
+		deadline: time::Instant,
+	) -> Result<Block, error::Error> {
+		/// If the block is full we will attempt to push at most
+		/// this number of transactions before quitting for real.
+		/// It allows us to increase block utilization.
+		const MAX_SKIPPED_TRANSACTIONS: usize = 8;
+
+		let mut block_builder = self.client.new_block_at(&self.parent_id, inherent_digests)?;
+
+		// We don't check the API versions any further here since the dispatch compatibility
+		// check should be enough.
+		for extrinsic in self.client.runtime_api()
+			.inherent_extrinsics_with_context(
+				&self.parent_id,
+				ExecutionContext::BlockConstruction,
+				inherent_data
+			)?
+		{
+			block_builder.push(extrinsic)?;
+		}
+
+		// proceed with transactions
+		let mut is_first = true;
+		let mut skipped = 0;
+		let mut unqueue_invalid = Vec::new();
+		let pending_iterator = self.transaction_pool.ready();
+
+		debug!("Attempting to push transactions from the pool.");
+		for pending in pending_iterator {
+			if (self.now)() > deadline {
+				debug!("Consensus deadline reached when pushing block transactions, proceeding with proposing.");
+				break;
+			}
+
+			trace!("[{:?}] Pushing to the block.", pending.hash);
+			match block_builder::BlockBuilder::push(&mut block_builder, pending.data.clone()) {
+				Ok(()) => {
+					debug!("[{:?}] Pushed to the block.", pending.hash);
+				}
+				Err(error::Error::ApplyExtrinsicFailed(e)) if e.exhausted_resources() => {
+					if is_first {
+						debug!("[{:?}] Invalid transaction: FullBlock on empty block", pending.hash);
+						unqueue_invalid.push(pending.hash.clone());
+					} else if skipped < MAX_SKIPPED_TRANSACTIONS {
+						skipped += 1;
+						debug!(
+							"Block seems full, but will try {} more transactions before quitting.",
+							MAX_SKIPPED_TRANSACTIONS - skipped,
+						);
+					} else {
+						debug!("Block is full, proceed with proposing.");
+						break;
+					}
+				}
+				Err(e) => {
+					debug!("[{:?}] Invalid transaction: {}", pending.hash, e);
+					unqueue_invalid.push(pending.hash.clone());
+				}
+			}
+
+			is_first = false;
+		}
+
+		self.transaction_pool.remove_invalid(&unqueue_invalid);
+
+		let block = block_builder.bake()?;
+
+		info!("Prepared block for proposing at {} [hash: {:?}; parent_hash: {}; extrinsics: [{}]]",
+			block.header().number(),
+			<Block as BlockT>::Hash::from(block.header().hash()),
+			block.header().parent_hash(),
+			block.extrinsics()
+				.iter()
+				.map(|xt| format!("{}", BlakeTwo256::hash_of(xt)))
+				.collect::<Vec<_>>()
+				.join(", ")
+		);
+		telemetry!(CONSENSUS_INFO; "prepared_block_for_proposing";
+			"number" => ?block.header().number(),
+			"hash" => ?<Block as BlockT>::Hash::from(block.header().hash()),
+		);
+
+		if Decode::decode(&mut block.encode().as_slice()).as_ref() != Ok(&block) {
+			error!("Failed to verify block encoding/decoding");
+		}
+
+		if let Err(err) = evaluation::evaluate_initial(&block, &self.parent_hash, self.parent_number) {
+			error!("Failed to evaluate authored block: {:?}", err);
+		}
+
+		Ok(block)
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	use std::cell::RefCell;
+	use consensus_common::{Environment, Proposer};
+	use test_client::{self, runtime::{Extrinsic, Transfer}, AccountKeyring};
+
+	fn extrinsic(nonce: u64) -> Extrinsic {
+		Transfer {
+			amount: Default::default(),
+			nonce,
+			from: AccountKeyring::Alice.into(),
+			to: Default::default(),
+		}.into_signed_tx()
+	}
+
+	#[test]
+	fn should_cease_building_block_when_deadline_is_reached() {
+		// given
+		let client = Arc::new(test_client::new());
+		let chain_api = transaction_pool::FullChainApi::new(client.clone());
+		let txpool = Arc::new(TransactionPool::new(Default::default(), chain_api));
+
+		futures::executor::block_on(
+			txpool.submit_at(&BlockId::number(0), vec![extrinsic(0), extrinsic(1)], false)
+		).unwrap();
+
+		let mut proposer_factory = ProposerFactory {
+			client: client.clone(),
+			transaction_pool: txpool.clone(),
+		};
+
+		let mut proposer = proposer_factory.init(
+			&client.header(&BlockId::number(0)).unwrap().unwrap(),
+		).unwrap();
+
+		// when
+		let cell = RefCell::new(time::Instant::now());
+		proposer.now = Box::new(move || {
+			let new = *cell.borrow() + time::Duration::from_secs(2);
+			cell.replace(new)
+		});
+		let deadline = time::Duration::from_secs(3);
+		let block = futures::executor::block_on(proposer.propose(Default::default(), Default::default(), deadline))
+			.unwrap();
+
+		// then
+		// block should have some extrinsics although we have some more in the pool.
+		assert_eq!(block.extrinsics().len(), 1);
+		assert_eq!(txpool.ready().count(), 2);
+	}
+}

substrate/client/basic-authorship/src/lib.rs

@@ -0,0 +1,58 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Basic implementation of block-authoring logic.
+//!
+//! # Example
+//!
+//! ```
+//! # use substrate_basic_authorship::ProposerFactory;
+//! # use consensus_common::{Environment, Proposer};
+//! # use sr_primitives::generic::BlockId;
+//! # use std::{sync::Arc, time::Duration};
+//! # use test_client::{self, runtime::{Extrinsic, Transfer}, AccountKeyring};
+//! # use transaction_pool::txpool::{self, Pool as TransactionPool};
+//! # let client = Arc::new(test_client::new());
+//! # let chain_api = transaction_pool::FullChainApi::new(client.clone());
+//! # let txpool = Arc::new(TransactionPool::new(Default::default(), chain_api));
+//! // The first step is to create a `ProposerFactory`.
+//! let mut proposer_factory = ProposerFactory {
+//! 	client: client.clone(),
+//! 	transaction_pool: txpool.clone(),
+//! };
+//!
+//! // From this factory, we create a `Proposer`.
+//! let mut proposer = proposer_factory.init(
+//! 	&client.header(&BlockId::number(0)).unwrap().unwrap(),
+//! ).unwrap();
+//!
+//! // This `Proposer` allows us to create a block proposition.
+//! // The proposer will grab transactions from the transaction pool, and put them into the block.
+//! let future = proposer.propose(
+//! 	Default::default(),
+//! 	Default::default(),
+//! 	Duration::from_secs(2)
+//! );
+//!
+//! // We wait until the proposition is performed.
+//! let block = futures::executor::block_on(future).unwrap();
+//! println!("Generated block: {:?}", block);
+//! ```
+//!
+
+mod basic_authorship;
+
+pub use crate::basic_authorship::{ProposerFactory, Proposer};

substrate/client/block-builder/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "substrate-block-builder"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+state-machine = { package = "substrate-state-machine", path = "../../primitives/state-machine" }
+sr-primitives = { path = "../../primitives/sr-primitives" }
+primitives = { package = "substrate-primitives", path = "../../primitives/core" }
+codec = { package = "parity-scale-codec", version = "1.0.6", features = ["derive"] }
+runtime_api = { package = "substrate-block-builder-runtime-api", path = "../../primitives/block-builder/runtime-api" }
+sr-api = { path = "../../primitives/sr-api" }
+

substrate/client/block-builder/src/lib.rs

@@ -0,0 +1,161 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Substrate block builder
+//!
+//! This crate provides the [`BlockBuilder`] utility and the corresponding runtime api
+//! [`BlockBuilder`](api::BlockBuilder).
+//!
+//! The block builder utility is used in the node as an abstraction over the runtime api to
+//! initialize a block, to push extrinsics and to finalize a block.
+
+#![warn(missing_docs)]
+
+use codec::Encode;
+
+use sr_primitives::{
+	generic::BlockId,
+	traits::{
+		Header as HeaderT, Hash, Block as BlockT, HashFor, ProvideRuntimeApi, ApiRef, DigestFor,
+		NumberFor, One,
+	},
+};
+
+use primitives::ExecutionContext;
+
+use state_machine::StorageProof;
+
+use sr_api::{Core, ApiExt, ApiErrorFor};
+
+pub use runtime_api::BlockBuilder as BlockBuilderApi;
+
+/// Error when the runtime failed to apply an extrinsic.
+pub struct ApplyExtrinsicFailed(pub sr_primitives::ApplyError);
+
+/// Utility for building new (valid) blocks from a stream of extrinsics.
+pub struct BlockBuilder<'a, Block: BlockT, A: ProvideRuntimeApi> {
+	header: Block::Header,
+	extrinsics: Vec<Block::Extrinsic>,
+	api: ApiRef<'a, A::Api>,
+	block_id: BlockId<Block>,
+}
+
+impl<'a, Block, A> BlockBuilder<'a, Block, A>
+where
+	Block: BlockT,
+	A: ProvideRuntimeApi + 'a,
+	A::Api: BlockBuilderApi<Block>,
+	ApiErrorFor<A, Block>: From<ApplyExtrinsicFailed>,
+{
+	/// Create a new instance of builder based on the given `parent_hash` and `parent_number`.
+	///
+	/// While proof recording is enabled, all accessed trie nodes are saved.
+	/// These recorded trie nodes can be used by a third party to prove the
+	/// output of this block builder without having access to the full storage.
+	pub fn new(
+		api: &'a A,
+		parent_hash: Block::Hash,
+		parent_number: NumberFor<Block>,
+		proof_recording: bool,
+		inherent_digests: DigestFor<Block>,
+	) -> Result<Self, ApiErrorFor<A, Block>> {
+		let header = <<Block as BlockT>::Header as HeaderT>::new(
+			parent_number + One::one(),
+			Default::default(),
+			Default::default(),
+			parent_hash,
+			inherent_digests,
+		);
+
+		let mut api = api.runtime_api();
+
+		if proof_recording {
+			api.record_proof();
+		}
+
+		let block_id = BlockId::Hash(parent_hash);
+
+		api.initialize_block_with_context(
+			&block_id, ExecutionContext::BlockConstruction, &header,
+		)?;
+
+		Ok(Self {
+			header,
+			extrinsics: Vec::new(),
+			api,
+			block_id,
+		})
+	}
+
+	/// Push onto the block's list of extrinsics.
+	///
+	/// This will ensure the extrinsic can be validly executed (by executing it);
+	pub fn push(&mut self, xt: <Block as BlockT>::Extrinsic) -> Result<(), ApiErrorFor<A, Block>> {
+		let block_id = &self.block_id;
+		let extrinsics = &mut self.extrinsics;
+
+		self.api.map_api_result(|api| {
+			match api.apply_extrinsic_with_context(
+				block_id,
+				ExecutionContext::BlockConstruction,
+				xt.clone()
+			)? {
+				Ok(_) => {
+					extrinsics.push(xt);
+					Ok(())
+				}
+				Err(e) => {
+					Err(ApplyExtrinsicFailed(e))?
+				}
+			}
+		})
+	}
+
+	/// Consume the builder to return a valid `Block` containing all pushed extrinsics.
+	pub fn bake(mut self) -> Result<Block, ApiErrorFor<A, Block>> {
+		self.bake_impl()?;
+		Ok(<Block as BlockT>::new(self.header, self.extrinsics))
+	}
+
+	fn bake_impl(&mut self) -> Result<(), ApiErrorFor<A, Block>> {
+		self.header = self.api.finalize_block_with_context(
+			&self.block_id, ExecutionContext::BlockConstruction
+		)?;
+
+		debug_assert_eq!(
+			self.header.extrinsics_root().clone(),
+			HashFor::<Block>::ordered_trie_root(
+				self.extrinsics.iter().map(Encode::encode).collect(),
+			),
+		);
+
+		Ok(())
+	}
+
+	/// Consume the builder to return a valid `Block` containing all pushed extrinsics
+	/// and the generated proof.
+	///
+	/// The proof will be `Some(_)`, if proof recording was enabled while creating
+	/// the block builder.
+	pub fn bake_and_extract_proof(mut self)
+		-> Result<(Block, Option<StorageProof>), ApiErrorFor<A, Block>>
+	{
+		self.bake_impl()?;
+
+		let proof = self.api.extract_proof();
+		Ok((<Block as BlockT>::new(self.header, self.extrinsics), proof))
+	}
+}

substrate/client/chain-spec/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "substrate-chain-spec"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+substrate-chain-spec-derive = { path = "./derive" }
+impl-trait-for-tuples = "0.1.3"
+network = { package = "substrate-network", path = "../network" }
+primitives = { package = "substrate-primitives", path = "../../primitives/core" }
+serde = { version = "1.0.101", features = ["derive"] }
+serde_json = "1.0.41"
+sr-primitives = { path = "../../primitives/sr-primitives" }
+tel = { package = "substrate-telemetry", path = "../telemetry" }

substrate/client/chain-spec/derive/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "substrate-chain-spec-derive"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+proc-macro-crate = "0.1.4"
+proc-macro2 = "1.0.6"
+quote = "1.0.2"
+syn = "1.0.7"
+
+[dev-dependencies]

substrate/client/chain-spec/derive/src/impls.rs

@@ -0,0 +1,190 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use proc_macro2::{Span, TokenStream};
+use quote::quote;
+use syn::{DeriveInput, Ident, Error};
+use proc_macro_crate::crate_name;
+
+const CRATE_NAME: &str = "substrate-chain-spec";
+const ATTRIBUTE_NAME: &str = "forks";
+
+/// Implements `Extension's` `Group` accessor.
+///
+/// The struct that derives this implementation will be usable within the `ChainSpec` file.
+/// The derive implements a by-type accessor method.
+pub fn extension_derive(ast: &DeriveInput) -> proc_macro::TokenStream {
+	derive(ast, |crate_name, name, generics: &syn::Generics, field_names, field_types, fields| {
+		let (impl_generics, ty_generics, where_clause) = generics.split_for_impl();
+		let forks = fields.named.iter().find_map(|f| {
+			if f.attrs.iter().any(|attr| attr.path.is_ident(ATTRIBUTE_NAME)) {
+				let typ = &f.ty;
+				Some(quote! { #typ })
+			} else {
+				None
+			}
+		}).unwrap_or_else(|| quote! { #crate_name::NoExtension });
+
+		quote! {
+			impl #impl_generics #crate_name::Extension for #name #ty_generics #where_clause {
+				type Forks = #forks;
+
+				fn get<T: 'static>(&self) -> Option<&T> {
+					use std::any::{Any, TypeId};
+
+					match TypeId::of::<T>() {
+						#( x if x == TypeId::of::<#field_types>() => Any::downcast_ref(&self.#field_names) ),*,
+						_ => None,
+					}
+				}
+			}
+		}
+	})
+}
+
+
+/// Implements required traits and creates `Fork` structs for `ChainSpec` custom parameter group.
+pub fn group_derive(ast: &DeriveInput) -> proc_macro::TokenStream {
+	derive(ast, |crate_name, name, generics: &syn::Generics, field_names, field_types, _fields| {
+		let (impl_generics, ty_generics, where_clause) = generics.split_for_impl();
+		let fork_name = Ident::new(&format!("{}Fork", name), Span::call_site());
+
+		let fork_fields = generate_fork_fields(&crate_name, &field_names, &field_types);
+		let to_fork = generate_base_to_fork(&fork_name, &field_names);
+		let combine_with = generate_combine_with(&field_names);
+		let to_base = generate_fork_to_base(name, &field_names);
+
+		quote! {
+			#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ChainSpecExtension)]
+			pub struct #fork_name #ty_generics #where_clause {
+				#fork_fields
+			}
+
+			impl #impl_generics #crate_name::Group for #name #ty_generics #where_clause {
+				type Fork = #fork_name #ty_generics;
+
+				fn to_fork(self) -> Self::Fork {
+					use #crate_name::Group;
+					#to_fork
+				}
+			}
+
+			impl #impl_generics #crate_name::Fork for #fork_name #ty_generics #where_clause {
+				type Base = #name #ty_generics;
+
+				fn combine_with(&mut self, other: Self) {
+					use #crate_name::Fork;
+					#combine_with
+				}
+
+				fn to_base(self) -> Option<Self::Base> {
+					use #crate_name::Fork;
+					#to_base
+				}
+			}
+		}
+	})
+}
+
+pub fn derive(
+	ast: &DeriveInput,
+	derive: impl Fn(
+		&Ident, &Ident, &syn::Generics, Vec<&Ident>, Vec<&syn::Type>, &syn::FieldsNamed,
+	) -> TokenStream,
+) -> proc_macro::TokenStream {
+	let err = || {
+		let err = Error::new(
+			Span::call_site(),
+			"ChainSpecGroup is only avaible for structs with named fields."
+		).to_compile_error();
+		quote!( #err ).into()
+	};
+
+	let data = match &ast.data {
+		syn::Data::Struct(ref data) => data,
+		_ => return err(),
+	};
+
+	let fields = match &data.fields {
+		syn::Fields::Named(ref named) => named,
+		_ => return err(),
+	};
+
+	const PROOF: &str = "CARGO_PKG_NAME always defined when compiling; qed";
+	let name = &ast.ident;
+	let crate_name = match crate_name(CRATE_NAME) {
+		Ok(chain_spec_name) => chain_spec_name,
+		Err(e) => if std::env::var("CARGO_PKG_NAME").expect(PROOF) == CRATE_NAME {
+			// we return the name of the crate here instead of `crate` to support doc tests.
+			CRATE_NAME.replace("-", "_")
+		} else {
+			let err = Error::new(Span::call_site(), &e).to_compile_error();
+			return quote!( #err ).into()
+		},
+	};
+	let crate_name = Ident::new(&crate_name, Span::call_site());
+	let field_names = fields.named.iter().flat_map(|x| x.ident.as_ref()).collect::<Vec<_>>();
+	let field_types = fields.named.iter().map(|x| &x.ty).collect::<Vec<_>>();
+
+	derive(&crate_name, name, &ast.generics, field_names, field_types, fields).into()
+}
+
+fn generate_fork_fields(
+	crate_name: &Ident,
+	names: &[&Ident],
+	types: &[&syn::Type],
+) -> TokenStream {
+	let crate_name = std::iter::repeat(crate_name);
+	quote! {
+		#( pub #names: Option<<#types as #crate_name::Group>::Fork>, )*
+	}
+}
+
+fn generate_base_to_fork(
+	fork_name: &Ident,
+	names: &[&Ident],
+) -> TokenStream {
+	let names2 = names.to_vec();
+
+	quote!{
+		#fork_name {
+			#( #names: Some(self.#names2.to_fork()), )*
+		}
+	}
+}
+
+fn generate_combine_with(
+	names: &[&Ident],
+) -> TokenStream {
+	let names2 = names.to_vec();
+
+	quote!{
+		#( self.#names.combine_with(other.#names2); )*
+	}
+}
+
+fn generate_fork_to_base(
+	fork: &Ident,
+	names: &[&Ident],
+) -> TokenStream {
+	let names2 = names.to_vec();
+
+	quote!{
+		Some(#fork {
+			#( #names: self.#names2?.to_base()?, )*
+		})
+	}
+}

substrate/client/chain-spec/derive/src/lib.rs

@@ -0,0 +1,39 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Macros to derive chain spec extension traits implementation.
+
+extern crate proc_macro;
+
+mod impls;
+
+use proc_macro::TokenStream;
+
+#[proc_macro_derive(ChainSpecGroup)]
+pub fn group_derive(input: TokenStream) -> TokenStream {
+	match syn::parse(input) {
+		Ok(ast) => impls::group_derive(&ast),
+		Err(e) => e.to_compile_error().into(),
+	}
+}
+
+#[proc_macro_derive(ChainSpecExtension, attributes(forks))]
+pub fn extensions_derive(input: TokenStream) -> TokenStream {
+	match syn::parse(input) {
+		Ok(ast) => impls::extension_derive(&ast),
+		Err(e) => e.to_compile_error().into(),
+	}
+}

substrate/client/chain-spec/res/chain_spec.json

@@ -0,0 +1,31 @@
+{
+	"name": "Flaming Fir",
+	"id": "flaming-fir",
+	"properties": {
+		"tokenDecimals": 15,
+		"tokenSymbol": "FIR"
+	},
+	"bootNodes": [
+		"/ip4/35.246.224.91/tcp/30333/p2p/QmSk5HQbn6LhUwDiNMseVUjuRYhEtYj4aUZ6WfWoGURpdV",
+		"/ip4/35.246.224.91/tcp/30334/ws/p2p/QmSk5HQbn6LhUwDiNMseVUjuRYhEtYj4aUZ6WfWoGURpdV",
+		"/ip4/35.246.210.11/tcp/30333/p2p/QmWv9Ww7znzgLFyCzf21SR6tUKXrmHCZH9KhebeH4gyE9f",
+		"/ip4/35.246.210.11/tcp/30334/ws/p2p/QmWv9Ww7znzgLFyCzf21SR6tUKXrmHCZH9KhebeH4gyE9f",
+		"/ip4/35.198.110.45/tcp/30333/p2p/QmTtcYKJho9vFmqtMA548QBSmLbmwAkBSiEKK3kWKfb6bJ",
+		"/ip4/35.198.110.45/tcp/30334/ws/p2p/QmTtcYKJho9vFmqtMA548QBSmLbmwAkBSiEKK3kWKfb6bJ",
+		"/ip4/35.198.114.154/tcp/30333/p2p/QmQJmDorK9c8KjMF5PdWiH2WGUXyzJtgTeJ55S5gggdju6",
+		"/ip4/35.198.114.154/tcp/30334/ws/p2p/QmQJmDorK9c8KjMF5PdWiH2WGUXyzJtgTeJ55S5gggdju6"
+	],
+	"telemetryEndpoints": [
+		["wss://telemetry.polkadot.io/submit/", 0]
+	],
+	"protocolId": "fir",
+	"consensusEngine": null,
+	"genesis": {
+		"raw": [
+			{
+				"0xb2029f8665aac509629f2d28cea790a3": "0x10f26cdb14b5aec7b2789fd5ca80f979cef3761897ae1f37ffb3e154cbcc1c26633919132b851ef0fd2dae42a7e734fe547af5a6b809006100f48944d7fae8e8ef00299981a2b92f878baaf5dbeba5c18d4e70f2a1fcd9c61b32ea18daf38f437800299981a2b92f878baaf5dbeba5c18d4e70f2a1fcd9c61b32ea18daf38f4378547ff0ab649283a7ae01dbc2eb73932eba2fb09075e9485ff369082a2ff38d655633b70b80a6c8bb16270f82cca6d56b27ed7b76c8fd5af2986a25a4788ce440482a3389a6cf42d8ed83888cfd920fec738ea30f97e44699ada7323f08c3380a482a3389a6cf42d8ed83888cfd920fec738ea30f97e44699ada7323f08c3380a68655684472b743e456907b398d3a44c113f189e56d1bbfd55e889e295dfde787932cff431e748892fa48e10c63c17d30f80ca42e4de3921e641249cd7fa3c2f482dbd7297a39fa145c570552249c2ca9dd47e281f0c500c971b59c9dcdcd82e482dbd7297a39fa145c570552249c2ca9dd47e281f0c500c971b59c9dcdcd82e9c7a2ee14e565db0c69f78c7b4cd839fbf52b607d867e9e9c5a79042898a0d129becad03e6dcac03cee07edebca5475314861492cdfc96a2144a67bbe96993326e7e4eb42cbd2e0ab4cae8708ce5509580b8c04d11f6758dbf686d50fe9f91066e7e4eb42cbd2e0ab4cae8708ce5509580b8c04d11f6758dbf686d50fe9f9106"
+			},
+			{}
+		]
+	}
+}

substrate/client/chain-spec/res/chain_spec2.json

@@ -0,0 +1,32 @@
+{
+	"name": "Flaming Fir",
+	"id": "flaming-fir",
+	"properties": {
+		"tokenDecimals": 15,
+		"tokenSymbol": "FIR"
+	},
+	"bootNodes": [
+		"/ip4/35.246.224.91/tcp/30333/p2p/QmSk5HQbn6LhUwDiNMseVUjuRYhEtYj4aUZ6WfWoGURpdV",
+		"/ip4/35.246.224.91/tcp/30334/ws/p2p/QmSk5HQbn6LhUwDiNMseVUjuRYhEtYj4aUZ6WfWoGURpdV",
+		"/ip4/35.246.210.11/tcp/30333/p2p/QmWv9Ww7znzgLFyCzf21SR6tUKXrmHCZH9KhebeH4gyE9f",
+		"/ip4/35.246.210.11/tcp/30334/ws/p2p/QmWv9Ww7znzgLFyCzf21SR6tUKXrmHCZH9KhebeH4gyE9f",
+		"/ip4/35.198.110.45/tcp/30333/p2p/QmTtcYKJho9vFmqtMA548QBSmLbmwAkBSiEKK3kWKfb6bJ",
+		"/ip4/35.198.110.45/tcp/30334/ws/p2p/QmTtcYKJho9vFmqtMA548QBSmLbmwAkBSiEKK3kWKfb6bJ",
+		"/ip4/35.198.114.154/tcp/30333/p2p/QmQJmDorK9c8KjMF5PdWiH2WGUXyzJtgTeJ55S5gggdju6",
+		"/ip4/35.198.114.154/tcp/30334/ws/p2p/QmQJmDorK9c8KjMF5PdWiH2WGUXyzJtgTeJ55S5gggdju6"
+	],
+	"telemetryEndpoints": [
+		["wss://telemetry.polkadot.io/submit/", 0]
+	],
+	"protocolId": "fir",
+	"consensusEngine": null,
+	"myProperty": "Test Extension",
+	"genesis": {
+		"raw": [
+			{
+				"0xb2029f8665aac509629f2d28cea790a3": "0x10f26cdb14b5aec7b2789fd5ca80f979cef3761897ae1f37ffb3e154cbcc1c26633919132b851ef0fd2dae42a7e734fe547af5a6b809006100f48944d7fae8e8ef00299981a2b92f878baaf5dbeba5c18d4e70f2a1fcd9c61b32ea18daf38f437800299981a2b92f878baaf5dbeba5c18d4e70f2a1fcd9c61b32ea18daf38f4378547ff0ab649283a7ae01dbc2eb73932eba2fb09075e9485ff369082a2ff38d655633b70b80a6c8bb16270f82cca6d56b27ed7b76c8fd5af2986a25a4788ce440482a3389a6cf42d8ed83888cfd920fec738ea30f97e44699ada7323f08c3380a482a3389a6cf42d8ed83888cfd920fec738ea30f97e44699ada7323f08c3380a68655684472b743e456907b398d3a44c113f189e56d1bbfd55e889e295dfde787932cff431e748892fa48e10c63c17d30f80ca42e4de3921e641249cd7fa3c2f482dbd7297a39fa145c570552249c2ca9dd47e281f0c500c971b59c9dcdcd82e482dbd7297a39fa145c570552249c2ca9dd47e281f0c500c971b59c9dcdcd82e9c7a2ee14e565db0c69f78c7b4cd839fbf52b607d867e9e9c5a79042898a0d129becad03e6dcac03cee07edebca5475314861492cdfc96a2144a67bbe96993326e7e4eb42cbd2e0ab4cae8708ce5509580b8c04d11f6758dbf686d50fe9f91066e7e4eb42cbd2e0ab4cae8708ce5509580b8c04d11f6758dbf686d50fe9f9106"
+			},
+			{}
+		]
+	}
+}

substrate/client/chain-spec/src/chain_spec.rs

@@ -0,0 +1,332 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Substrate chain configurations.
+
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::fs::File;
+use std::path::PathBuf;
+use std::rc::Rc;
+use serde::{Serialize, Deserialize};
+use primitives::storage::{StorageKey, StorageData};
+use sr_primitives::{BuildStorage, StorageOverlay, ChildrenStorageOverlay};
+use serde_json as json;
+use crate::RuntimeGenesis;
+use network::Multiaddr;
+use tel::TelemetryEndpoints;
+
+enum GenesisSource<G> {
+	File(PathBuf),
+	Binary(Cow<'static, [u8]>),
+	Factory(Rc<dyn Fn() -> G>),
+}
+
+impl<G> Clone for GenesisSource<G> {
+	fn clone(&self) -> Self {
+		match *self {
+			GenesisSource::File(ref path) => GenesisSource::File(path.clone()),
+			GenesisSource::Binary(ref d) => GenesisSource::Binary(d.clone()),
+			GenesisSource::Factory(ref f) => GenesisSource::Factory(f.clone()),
+		}
+	}
+}
+
+impl<G: RuntimeGenesis> GenesisSource<G> {
+	fn resolve(&self) -> Result<Genesis<G>, String> {
+		#[derive(Serialize, Deserialize)]
+		struct GenesisContainer<G> {
+			genesis: Genesis<G>,
+		}
+
+		match self {
+			GenesisSource::File(path) => {
+				let file = File::open(path)
+					.map_err(|e| format!("Error opening spec file: {}", e))?;
+				let genesis: GenesisContainer<G> = json::from_reader(file)
+					.map_err(|e| format!("Error parsing spec file: {}", e))?;
+				Ok(genesis.genesis)
+			},
+			GenesisSource::Binary(buf) => {
+				let genesis: GenesisContainer<G> = json::from_reader(buf.as_ref())
+					.map_err(|e| format!("Error parsing embedded file: {}", e))?;
+				Ok(genesis.genesis)
+			},
+			GenesisSource::Factory(f) => Ok(Genesis::Runtime(f())),
+		}
+	}
+}
+
+impl<'a, G: RuntimeGenesis, E> BuildStorage for &'a ChainSpec<G, E> {
+	fn build_storage(&self) -> Result<(StorageOverlay, ChildrenStorageOverlay), String> {
+		match self.genesis.resolve()? {
+			Genesis::Runtime(gc) => gc.build_storage(),
+			Genesis::Raw(map, children_map) => Ok((
+				map.into_iter().map(|(k, v)| (k.0, v.0)).collect(),
+				children_map.into_iter().map(|(sk, map)| (
+					sk.0,
+					map.into_iter().map(|(k, v)| (k.0, v.0)).collect(),
+				)).collect(),
+			)),
+		}
+	}
+
+	fn assimilate_storage(
+		&self,
+		_: &mut (StorageOverlay, ChildrenStorageOverlay)
+	) -> Result<(), String> {
+		Err("`assimilate_storage` not implemented for `ChainSpec`.".into())
+	}
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+#[serde(deny_unknown_fields)]
+enum Genesis<G> {
+	Runtime(G),
+	Raw(
+		HashMap<StorageKey, StorageData>,
+		HashMap<StorageKey, HashMap<StorageKey, StorageData>>,
+	),
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+#[serde(deny_unknown_fields)]
+struct ChainSpecFile<E> {
+	pub name: String,
+	pub id: String,
+	pub boot_nodes: Vec<String>,
+	pub telemetry_endpoints: Option<TelemetryEndpoints>,
+	pub protocol_id: Option<String>,
+	pub properties: Option<Properties>,
+	#[serde(flatten)]
+	pub extensions: E,
+	// Never used, left only for backward compatibility.
+	consensus_engine: (),
+	#[serde(skip_serializing)]
+	genesis: serde::de::IgnoredAny,
+}
+
+/// Arbitrary properties defined in chain spec as a JSON object
+pub type Properties = json::map::Map<String, json::Value>;
+
+/// A type denoting empty extensions.
+///
+/// We use `Option` here since `()` is not flattenable by serde.
+pub type NoExtension = Option<()>;
+
+/// A configuration of a chain. Can be used to build a genesis block.
+pub struct ChainSpec<G, E = NoExtension> {
+	spec: ChainSpecFile<E>,
+	genesis: GenesisSource<G>,
+}
+
+impl<G, E: Clone> Clone for ChainSpec<G, E> {
+	fn clone(&self) -> Self {
+		ChainSpec {
+			spec: self.spec.clone(),
+			genesis: self.genesis.clone(),
+		}
+	}
+}
+
+impl<G, E> ChainSpec<G, E> {
+	/// A list of bootnode addresses.
+	pub fn boot_nodes(&self) -> &[String] {
+		&self.spec.boot_nodes
+	}
+
+	/// Spec name.
+	pub fn name(&self) -> &str {
+		&self.spec.name
+	}
+
+	/// Spec id.
+	pub fn id(&self) -> &str {
+		&self.spec.id
+	}
+
+	/// Telemetry endpoints (if any)
+	pub fn telemetry_endpoints(&self) -> &Option<TelemetryEndpoints> {
+		&self.spec.telemetry_endpoints
+	}
+
+	/// Network protocol id.
+	pub fn protocol_id(&self) -> Option<&str> {
+		self.spec.protocol_id.as_ref().map(String::as_str)
+	}
+
+	/// Additional loosly-typed properties of the chain.
+	///
+	/// Returns an empty JSON object if 'properties' not defined in config
+	pub fn properties(&self) -> Properties {
+		self.spec.properties.as_ref().unwrap_or(&json::map::Map::new()).clone()
+	}
+
+	/// Add a bootnode to the list.
+	pub fn add_boot_node(&mut self, addr: Multiaddr) {
+		self.spec.boot_nodes.push(addr.to_string())
+	}
+
+	/// Returns a reference to defined chain spec extensions.
+	pub fn extensions(&self) -> &E {
+		&self.spec.extensions
+	}
+
+	/// Create hardcoded spec.
+	pub fn from_genesis<F: Fn() -> G + 'static>(
+		name: &str,
+		id: &str,
+		constructor: F,
+		boot_nodes: Vec<String>,
+		telemetry_endpoints: Option<TelemetryEndpoints>,
+		protocol_id: Option<&str>,
+		properties: Option<Properties>,
+		extensions: E,
+	) -> Self {
+		let spec = ChainSpecFile {
+			name: name.to_owned(),
+			id: id.to_owned(),
+			boot_nodes: boot_nodes,
+			telemetry_endpoints,
+			protocol_id: protocol_id.map(str::to_owned),
+			properties,
+			extensions,
+			consensus_engine: (),
+			genesis: Default::default(),
+		};
+
+		ChainSpec {
+			spec,
+			genesis: GenesisSource::Factory(Rc::new(constructor)),
+		}
+	}
+}
+
+impl<G, E: serde::de::DeserializeOwned> ChainSpec<G, E> {
+	/// Parse json content into a `ChainSpec`
+	pub fn from_json_bytes(json: impl Into<Cow<'static, [u8]>>) -> Result<Self, String> {
+		let json = json.into();
+		let spec = json::from_slice(json.as_ref())
+			.map_err(|e| format!("Error parsing spec file: {}", e))?;
+		Ok(ChainSpec {
+			spec,
+			genesis: GenesisSource::Binary(json),
+		})
+	}
+
+	/// Parse json file into a `ChainSpec`
+	pub fn from_json_file(path: PathBuf) -> Result<Self, String> {
+		let file = File::open(&path)
+			.map_err(|e| format!("Error opening spec file: {}", e))?;
+		let spec = json::from_reader(file)
+			.map_err(|e| format!("Error parsing spec file: {}", e))?;
+		Ok(ChainSpec {
+			spec,
+			genesis: GenesisSource::File(path),
+		})
+	}
+}
+
+impl<G: RuntimeGenesis, E: serde::Serialize> ChainSpec<G, E> {
+	/// Dump to json string.
+	pub fn to_json(self, raw: bool) -> Result<String, String> {
+		#[derive(Serialize, Deserialize)]
+		struct Container<G, E> {
+			#[serde(flatten)]
+			spec: ChainSpecFile<E>,
+			genesis: Genesis<G>,
+
+		};
+		let genesis = match (raw, self.genesis.resolve()?) {
+			(true, Genesis::Runtime(g)) => {
+				let storage = g.build_storage()?;
+				let top = storage.0.into_iter()
+					.map(|(k, v)| (StorageKey(k), StorageData(v)))
+					.collect();
+				let children = storage.1.into_iter()
+					.map(|(sk, child)| (
+							StorageKey(sk),
+							child.into_iter()
+								.map(|(k, v)| (StorageKey(k), StorageData(v)))
+								.collect(),
+					))
+					.collect();
+
+				Genesis::Raw(top, children)
+			},
+			(_, genesis) => genesis,
+		};
+		let spec = Container {
+			spec: self.spec,
+			genesis,
+		};
+		json::to_string_pretty(&spec)
+			.map_err(|e| format!("Error generating spec json: {}", e))
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[derive(Debug, Serialize, Deserialize)]
+	struct Genesis(HashMap<String, String>);
+
+	impl BuildStorage for Genesis {
+		fn assimilate_storage(
+			&self,
+			storage: &mut (StorageOverlay, ChildrenStorageOverlay),
+		) -> Result<(), String> {
+			storage.0.extend(
+				self.0.iter().map(|(a, b)| (a.clone().into_bytes(), b.clone().into_bytes()))
+			);
+			Ok(())
+		}
+	}
+
+	type TestSpec = ChainSpec<Genesis>;
+
+	#[test]
+	fn should_deserailize_example_chain_spec() {
+		let spec1 = TestSpec::from_json_bytes(Cow::Owned(
+			include_bytes!("../res/chain_spec.json").to_vec()
+		)).unwrap();
+		let spec2 = TestSpec::from_json_file(
+			PathBuf::from("./res/chain_spec.json")
+		).unwrap();
+
+		assert_eq!(spec1.to_json(false), spec2.to_json(false));
+	}
+
+	#[derive(Debug, Serialize, Deserialize)]
+	#[serde(rename_all = "camelCase")]
+	struct Extension1 {
+		my_property: String,
+	}
+
+	type TestSpec2 = ChainSpec<Genesis, Extension1>;
+
+	#[test]
+	fn should_deserialize_chain_spec_with_extensions() {
+		let spec = TestSpec2::from_json_bytes(Cow::Owned(
+			include_bytes!("../res/chain_spec2.json").to_vec()
+		)).unwrap();
+
+		assert_eq!(spec.extensions().my_property, "Test Extension");
+	}
+}

substrate/client/chain-spec/src/extension.rs

@@ -0,0 +1,363 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Chain Spec extensions helpers.
+
+use std::fmt::Debug;
+use std::collections::BTreeMap;
+
+use serde::{Serialize, Deserialize, de::DeserializeOwned};
+
+/// A `ChainSpec` extension.
+///
+/// This trait is implemented automatically by `ChainSpecGroup` macro.
+pub trait Group: Clone + Sized {
+	/// An associated type containing fork definition.
+	type Fork: Fork<Base=Self>;
+
+	/// Convert to fork type.
+	fn to_fork(self) -> Self::Fork;
+}
+
+/// A `ChainSpec` extension fork definition.
+///
+/// Basically should look the same as `Group`, but
+/// all parameters are optional. This allows changing
+/// only one parameter as part of the fork.
+/// The forks can be combined (summed up) to specify
+/// a complete set of parameters
+pub trait Fork: Serialize + DeserializeOwned + Clone + Sized {
+	/// A base `Group` type.
+	type Base: Group<Fork=Self>;
+
+	/// Combine with another struct.
+	///
+	/// All parameters set in `other` should override the
+	/// ones in the current struct.
+	fn combine_with(&mut self, other: Self);
+
+	/// Attempt to convert to the base type if all parameters are set.
+	fn to_base(self) -> Option<Self::Base>;
+}
+
+macro_rules! impl_trivial {
+	() => {};
+	($A : ty) => {
+		impl_trivial!($A ,);
+	};
+	($A : ty , $( $B : ty ),*) => {
+		impl_trivial!($( $B ),*);
+
+		impl Group for $A {
+			type Fork = $A;
+
+			fn to_fork(self) -> Self::Fork {
+				self
+			}
+		}
+
+		impl Fork for $A {
+			type Base = $A;
+
+			fn combine_with(&mut self, other: Self) {
+				*self = other;
+			}
+
+			fn to_base(self) -> Option<Self::Base> {
+				Some(self)
+			}
+		}
+	}
+}
+
+impl_trivial!((), u8, u16, u32, u64, usize, String, Vec<u8>);
+
+impl<T: Group> Group for Option<T> {
+	type Fork = Option<T::Fork>;
+
+	fn to_fork(self) -> Self::Fork {
+		self.map(|a| a.to_fork())
+	}
+}
+
+impl<T: Fork> Fork for Option<T> {
+	type Base = Option<T::Base>;
+
+	fn combine_with(&mut self, other: Self) {
+		*self = match (self.take(), other) {
+			(Some(mut a), Some(b)) => {
+				a.combine_with(b);
+				Some(a)
+			},
+			(a, b) => a.or(b),
+		};
+	}
+
+	fn to_base(self) -> Option<Self::Base> {
+		self.map(|x| x.to_base())
+	}
+}
+
+/// A collection of `ChainSpec` extensions.
+///
+/// This type can be passed around and allows the core
+/// modules to request a strongly-typed, but optional configuration.
+pub trait Extension: Serialize + DeserializeOwned + Clone {
+	type Forks: IsForks;
+
+	/// Get an extension of specific type.
+	fn get<T: 'static>(&self) -> Option<&T>;
+
+	/// Get forkable extensions of specific type.
+	fn forks<BlockNumber, T>(&self) -> Option<Forks<BlockNumber, T>> where
+		BlockNumber: Ord + Clone + 'static,
+		T: Group + 'static,
+		<Self::Forks as IsForks>::Extension: Extension,
+		<<Self::Forks as IsForks>::Extension as Group>::Fork: Extension,
+	{
+		self.get::<Forks<BlockNumber, <Self::Forks as IsForks>::Extension>>()?
+			.for_type()
+	}
+}
+
+impl Extension for crate::NoExtension {
+	type Forks = Self;
+
+	fn get<T: 'static>(&self) -> Option<&T> { None }
+}
+
+pub trait IsForks {
+	type BlockNumber: Ord + 'static;
+	type Extension: Group + 'static;
+}
+
+impl IsForks for Option<()> {
+	type BlockNumber = u64;
+	type Extension = Self;
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
+#[serde(deny_unknown_fields)]
+pub struct Forks<BlockNumber: Ord, T: Group> {
+	forks: BTreeMap<BlockNumber, T::Fork>,
+	#[serde(flatten)]
+	base: T,
+}
+
+impl<B: Ord, T: Group + Default> Default for Forks<B, T> {
+	fn default() -> Self {
+		Self {
+			base: Default::default(),
+			forks: Default::default(),
+		}
+	}
+}
+
+impl<B: Ord, T: Group> Forks<B, T> where
+	T::Fork: Debug,
+{
+	/// Create new fork definition given the base and the forks.
+	pub fn new(base: T, forks: BTreeMap<B, T::Fork>) -> Self {
+		Self { base, forks }
+	}
+
+	/// Return a set of parameters for `Group` including all forks up to `block` (inclusive).
+	pub fn at_block(&self, block: B) -> T {
+		let mut start = self.base.clone().to_fork();
+
+		for (_, fork) in self.forks.range(..=block) {
+			start.combine_with(fork.clone());
+		}
+
+		start
+			.to_base()
+			.expect("We start from the `base` object, so it's always fully initialized; qed")
+	}
+}
+
+impl<B, T> IsForks for Forks<B, T> where
+	B: Ord + 'static,
+	T: Group + 'static,
+{
+	type BlockNumber = B;
+	type Extension = T;
+}
+
+impl<B: Ord + Clone, T: Group + Extension> Forks<B, T> where
+	T::Fork: Extension,
+{
+	/// Get forks definition for a subset of this extension.
+	///
+	/// Returns the `Forks` struct, but limited to a particular type
+	/// within the extension.
+	pub fn for_type<X>(&self) -> Option<Forks<B, X>> where
+		X: Group + 'static,
+	{
+		let base = self.base.get::<X>()?.clone();
+		let forks = self.forks.iter().filter_map(|(k, v)| {
+			Some((k.clone(), v.get::<Option<X::Fork>>()?.clone()?))
+		}).collect();
+
+		Some(Forks {
+			base,
+			forks,
+		})
+	}
+}
+
+impl<B, E> Extension for Forks<B, E> where
+	B: Serialize + DeserializeOwned + Ord + Clone + 'static,
+	E: Extension + Group + 'static,
+{
+	type Forks = Self;
+
+	fn get<T: 'static>(&self) -> Option<&T> {
+		use std::any::{TypeId, Any};
+
+		match TypeId::of::<T>() {
+			x if x == TypeId::of::<E>() => Any::downcast_ref(&self.base),
+			_ => self.base.get(),
+		}
+	}
+
+	fn forks<BlockNumber, T>(&self) -> Option<Forks<BlockNumber, T>> where
+		BlockNumber: Ord + Clone + 'static,
+		T: Group + 'static,
+		<Self::Forks as IsForks>::Extension: Extension,
+		<<Self::Forks as IsForks>::Extension as Group>::Fork: Extension,
+	{
+		use std::any::{TypeId, Any};
+
+		if TypeId::of::<BlockNumber>() == TypeId::of::<B>() {
+			Any::downcast_ref(&self.for_type::<T>()?).cloned()
+		} else {
+			self.get::<Forks<BlockNumber, <Self::Forks as IsForks>::Extension>>()?
+				.for_type()
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use substrate_chain_spec_derive::{ChainSpecGroup, ChainSpecExtension};
+	// Make the proc macro work for tests and doc tests.
+	use crate as substrate_chain_spec;
+
+	#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ChainSpecGroup)]
+	#[serde(deny_unknown_fields)]
+	pub struct Extension1 {
+		pub test: u64,
+	}
+
+	#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ChainSpecGroup)]
+	#[serde(deny_unknown_fields)]
+	pub struct Extension2 {
+		pub test: u8,
+	}
+
+	#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ChainSpecGroup, ChainSpecExtension)]
+	#[serde(deny_unknown_fields)]
+	pub struct Extensions {
+		pub ext1: Extension1,
+		pub ext2: Extension2,
+	}
+
+	#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ChainSpecExtension)]
+	#[serde(deny_unknown_fields)]
+	pub struct Ext2 {
+		#[serde(flatten)]
+		ext1: Extension1,
+		#[forks]
+		forkable: Forks<u64, Extensions>,
+	}
+
+	#[test]
+	fn forks_should_work_correctly() {
+		use super::Extension as _ ;
+
+		let ext: Ext2 = serde_json::from_str(r#"
+{
+	"test": 11,
+	"forkable": {
+		"ext1": {
+			"test": 15
+		},
+		"ext2": {
+			"test": 123
+		},
+		"forks": {
+			"1": {
+				"ext1": { "test": 5 }
+			},
+			"2": {
+				"ext2": { "test": 5 }
+			},
+			"5": {
+				"ext2": { "test": 1 }
+			}
+		}
+	}
+}
+		"#).unwrap();
+
+		assert_eq!(ext.get::<Extension1>(), Some(&Extension1 {
+			test: 11
+		}));
+
+		// get forks definition
+		let forks = ext.get::<Forks<u64, Extensions>>().unwrap();
+		assert_eq!(forks.at_block(0), Extensions {
+			ext1: Extension1 { test: 15 },
+			ext2: Extension2 { test: 123 },
+		});
+		assert_eq!(forks.at_block(1), Extensions {
+			ext1: Extension1 { test: 5 },
+			ext2: Extension2 { test: 123 },
+		});
+		assert_eq!(forks.at_block(2), Extensions {
+			ext1: Extension1 { test: 5 },
+			ext2: Extension2 { test: 5 },
+		});
+		assert_eq!(forks.at_block(4), Extensions {
+			ext1: Extension1 { test: 5 },
+			ext2: Extension2 { test: 5 },
+		});
+		assert_eq!(forks.at_block(5), Extensions {
+			ext1: Extension1 { test: 5 },
+			ext2: Extension2 { test: 1 },
+		});
+		assert_eq!(forks.at_block(10), Extensions {
+			ext1: Extension1 { test: 5 },
+			ext2: Extension2 { test: 1 },
+		});
+		assert!(forks.at_block(10).get::<Extension2>().is_some());
+
+		// filter forks for `Extension2`
+		let ext2 = forks.for_type::<Extension2>().unwrap();
+		assert_eq!(ext2.at_block(0), Extension2 { test: 123 });
+		assert_eq!(ext2.at_block(2), Extension2 { test: 5 });
+		assert_eq!(ext2.at_block(10), Extension2 { test: 1 });
+
+		// make sure that it can return forks correctly
+		let ext2_2 = forks.forks::<u64, Extension2>().unwrap();
+		assert_eq!(ext2, ext2_2);
+
+		// also ext should be able to return forks correctly:
+		let ext2_3 = ext.forks::<u64, Extension2>().unwrap();
+		assert_eq!(ext2_2, ext2_3);
+	}
+}

substrate/client/chain-spec/src/lib.rs

@@ -0,0 +1,123 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Substrate chain configurations.
+//!
+//! This crate contains structs and utilities to declare
+//! a runtime-specific configuration file (a.k.a chain spec).
+//!
+//! Basic chain spec type containing all required parameters is
+//! [`ChainSpec`](./struct.ChainSpec.html). It can be extended with
+//! additional options that contain configuration specific to your chain.
+//! Usually the extension is going to be an amalgamate of types exposed
+//! by Substrate core modules. To allow the core modules to retrieve
+//! their configuration from your extension you should use `ChainSpecExtension`
+//! macro exposed by this crate.
+//!
+//! ```rust
+//! use std::collections::HashMap;
+//! use serde::{Serialize, Deserialize};
+//! use substrate_chain_spec::{ChainSpec, ChainSpecExtension};
+//!
+//! #[derive(Clone, Debug, Serialize, Deserialize, ChainSpecExtension)]
+//! pub struct MyExtension {
+//!		pub known_blocks: HashMap<u64, String>,
+//! }
+//!
+//! pub type MyChainSpec<G> = ChainSpec<G, MyExtension>;
+//! ```
+//!
+//! Some parameters may require different values depending on the
+//! current blockchain height (a.k.a. forks). You can use `ChainSpecGroup`
+//! macro and provided [`Forks`](./struct.Forks.html) structure to put
+//! such parameters to your chain spec.
+//! This will allow to override a single parameter starting at specific
+//! block number.
+//!
+//! ```rust
+//! use serde::{Serialize, Deserialize};
+//! use substrate_chain_spec::{Forks, ChainSpec, ChainSpecGroup, ChainSpecExtension};
+//!
+//! #[derive(Clone, Debug, Serialize, Deserialize, ChainSpecGroup)]
+//! pub struct ClientParams {
+//!		max_block_size: usize,
+//!		max_extrinsic_size: usize,
+//! }
+//!
+//! #[derive(Clone, Debug, Serialize, Deserialize, ChainSpecGroup)]
+//! pub struct PoolParams {
+//!		max_transaction_size: usize,
+//! }
+//!
+//! #[derive(Clone, Debug, Serialize, Deserialize, ChainSpecGroup, ChainSpecExtension)]
+//! pub struct Extension {
+//!		pub client: ClientParams,
+//!		pub pool: PoolParams,
+//! }
+//!
+//! pub type BlockNumber = u64;
+//!
+//! /// A chain spec supporting forkable `ClientParams`.
+//! pub type MyChainSpec1<G> = ChainSpec<G, Forks<BlockNumber, ClientParams>>;
+//!
+//! /// A chain spec supporting forkable `Extension`.
+//! pub type MyChainSpec2<G> = ChainSpec<G, Forks<BlockNumber, Extension>>;
+//! ```
+//!
+//! It's also possible to have a set of parameters that is allowed to change
+//! with block numbers (i.e. is forkable), and another set that is not subject to changes.
+//! This is also possible by declaring an extension that contains `Forks` within it.
+//!
+//!
+//! ```rust
+//! use serde::{Serialize, Deserialize};
+//! use substrate_chain_spec::{Forks, ChainSpec, ChainSpecGroup, ChainSpecExtension};
+//!
+//! #[derive(Clone, Debug, Serialize, Deserialize, ChainSpecGroup)]
+//! pub struct ClientParams {
+//!		max_block_size: usize,
+//!		max_extrinsic_size: usize,
+//! }
+//!
+//! #[derive(Clone, Debug, Serialize, Deserialize, ChainSpecGroup)]
+//! pub struct PoolParams {
+//!		max_transaction_size: usize,
+//! }
+//!
+//! #[derive(Clone, Debug, Serialize, Deserialize, ChainSpecExtension)]
+//! pub struct Extension {
+//!		pub client: ClientParams,
+//!		#[forks]
+//!		pub pool: Forks<u64, PoolParams>,
+//! }
+//!
+//! pub type MyChainSpec<G> = ChainSpec<G, Extension>;
+//! ```
+
+
+mod chain_spec;
+mod extension;
+
+pub use chain_spec::{ChainSpec, Properties, NoExtension};
+pub use extension::{Group, Fork, Forks, Extension};
+pub use substrate_chain_spec_derive::{ChainSpecExtension, ChainSpecGroup};
+
+use serde::{Serialize, de::DeserializeOwned};
+use sr_primitives::BuildStorage;
+
+/// A set of traits for the runtime genesis config.
+pub trait RuntimeGenesis: Serialize + DeserializeOwned + BuildStorage {}
+impl<T: Serialize + DeserializeOwned + BuildStorage> RuntimeGenesis for T {}

substrate/client/cli/Cargo.toml

@@ -0,0 +1,45 @@
+[package]
+name = "substrate-cli"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Substrate CLI interface."
+edition = "2018"
+
+[dependencies]
+clap = "2.33.0"
+derive_more = "0.15.0"
+env_logger = "0.7.0"
+log = "0.4.8"
+atty = "0.2.13"
+regex = "1.3.1"
+time = "0.1.42"
+ansi_term = "0.12.1"
+lazy_static = "1.4.0"
+app_dirs = "1.2.1"
+tokio = "0.1.22"
+futures = "0.1.29"
+futures03 = { package = "futures-preview", version = "=0.3.0-alpha.19", features = ["compat"] }
+fdlimit = "0.1.1"
+exit-future = "0.1.4"
+serde_json = "1.0.41"
+panic-handler = { package = "substrate-panic-handler", path = "../../primitives/panic-handler" }
+client-api = { package = "substrate-client-api", path = "../api" }
+header-metadata = { package = "substrate-header-metadata", path = "../header-metadata" }
+network = { package = "substrate-network", path = "../network" }
+sr-primitives = { path = "../../primitives/sr-primitives" }
+primitives = { package = "substrate-primitives", path = "../../primitives/core" }
+service = { package = "substrate-service", path = "../service", default-features = false }
+state-machine = { package = "substrate-state-machine", path = "../../primitives/state-machine" }
+substrate-telemetry = { path = "../telemetry" }
+keyring = { package = "substrate-keyring", path = "../../primitives/keyring" }
+names = "0.11.0"
+structopt = "0.3.3"
+rpassword = "4.0.1"
+
+[dev-dependencies]
+tempdir = "0.3.7"
+
+[features]
+wasmtime = [
+    "service/wasmtime",
+]

substrate/client/cli/README.adoc

@@ -0,0 +1,6 @@
+
+= Substrate CLI
+
+Substrate CLI library
+
+include::doc/shell-completion.adoc[]

substrate/client/cli/src/error.rs

@@ -0,0 +1,64 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Initialization errors.
+
+use client_api;
+
+/// Result type alias for the CLI.
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Error type for the CLI.
+#[derive(Debug, derive_more::Display, derive_more::From)]
+pub enum Error {
+	/// Io error
+	Io(std::io::Error),
+	/// Cli error
+	Cli(clap::Error),
+	/// Service error
+	Service(service::Error),
+	/// Client error
+	Client(client_api::error::Error),
+	/// Input error
+	Input(String),
+	/// Invalid listen multiaddress
+	#[display(fmt="Invalid listen multiaddress")]
+	InvalidListenMultiaddress,
+	/// Other uncategorized error.
+	Other(String),
+}
+
+/// Must be implemented explicitly because `derive_more` won't generate this
+/// case due to conflicting derive for `Other(String)`.
+impl std::convert::From<String> for Error {
+	fn from(s: String) -> Error {
+		Error::Input(s)
+	}
+}
+
+impl std::error::Error for Error {
+	fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+		match self {
+			Error::Io(ref err) => Some(err),
+			Error::Cli(ref err) => Some(err),
+			Error::Service(ref err) => Some(err),
+			Error::Client(ref err) => Some(err),
+			Error::Input(_) => None,
+			Error::InvalidListenMultiaddress => None,
+			Error::Other(_) => None,
+		}
+	}
+}

substrate/client/cli/src/execution_strategy.rs

@@ -0,0 +1,35 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+#![allow(missing_docs)]
+
+use structopt::clap::arg_enum;
+
+arg_enum! {
+	/// How to execute blocks
+	#[derive(Debug, Clone, Copy)]
+	pub enum ExecutionStrategy {
+		// Execute with native build (if available, WebAssembly otherwise).
+		Native,
+		// Only execute with the WebAssembly build.
+		Wasm,
+		// Execute with both native (where available) and WebAssembly builds.
+		Both,
+		// Execute with the native build if possible; if it fails, then execute with WebAssembly.
+		NativeElseWasm,
+	}
+}
+

substrate/client/cli/src/informant.rs

@@ -0,0 +1,82 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Console informant. Prints sync progress and block events. Runs on the calling thread.
+
+use client_api::BlockchainEvents;
+use futures::{Future, Stream};
+use futures03::{StreamExt as _, TryStreamExt as _};
+use log::{info, warn};
+use sr_primitives::traits::Header;
+use service::AbstractService;
+use std::time::Duration;
+
+mod display;
+
+/// Creates an informant in the form of a `Future` that must be polled regularly.
+pub fn build(service: &impl AbstractService) -> impl Future<Item = (), Error = ()> {
+	let client = service.client();
+
+	let mut display = display::InformantDisplay::new();
+
+	let display_notifications = service
+		.network_status(Duration::from_millis(5000))
+		.for_each(move |(net_status, _)| {
+			let info = client.info();
+			display.display(&info, net_status);
+			Ok(())
+		});
+
+	let client = service.client();
+	let mut last_best = {
+		let info = client.info();
+		Some((info.chain.best_number, info.chain.best_hash))
+	};
+
+	let display_block_import = client.import_notification_stream().map(|v| Ok::<_, ()>(v)).compat().for_each(move |n| {
+		// detect and log reorganizations.
+		if let Some((ref last_num, ref last_hash)) = last_best {
+			if n.header.parent_hash() != last_hash && n.is_new_best  {
+				let maybe_ancestor = header_metadata::lowest_common_ancestor(
+					&*client,
+					last_hash.clone(),
+					n.hash,
+				);
+
+				match maybe_ancestor {
+					Ok(ref ancestor) if ancestor.hash != *last_hash => info!(
+						"Reorg from #{},{} to #{},{}, common ancestor #{},{}",
+						last_num, last_hash,
+						n.header.number(), n.hash,
+						ancestor.number, ancestor.hash,
+					),
+					Ok(_) => {},
+					Err(e) => warn!("Error computing tree route: {}", e),
+				}
+			}
+		}
+
+		if n.is_new_best {
+			last_best = Some((n.header.number().clone(), n.hash.clone()));
+		}
+
+		info!(target: "substrate", "Imported #{} ({})", n.header.number(), n.hash);
+		Ok(())
+	});
+
+	display_notifications.join(display_block_import)
+		.map(|((), ())| ())
+}

substrate/client/cli/src/informant/display.rs

@@ -0,0 +1,149 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use ansi_term::Colour;
+use client_api::ClientInfo;
+use log::info;
+use network::SyncState;
+use sr_primitives::traits::{Block as BlockT, CheckedDiv, NumberFor, Zero, Saturating};
+use service::NetworkStatus;
+use std::{convert::{TryFrom, TryInto}, fmt, time};
+
+/// State of the informant display system.
+///
+/// This is the system that handles the line that gets regularly printed and that looks something
+/// like:
+///
+/// > Syncing  5.4 bps, target=#531028 (4 peers), best: #90683 (0x4ca8…51b8),
+/// >  finalized #360 (0x6f24…a38b), ⬇ 5.5kiB/s ⬆ 0.9kiB/s
+///
+/// # Usage
+///
+/// Call `InformantDisplay::new` to initialize the state, then regularly call `display` with the
+/// information to display.
+///
+pub struct InformantDisplay<B: BlockT> {
+	/// Head of chain block number from the last time `display` has been called.
+	/// `None` if `display` has never been called.
+	last_number: Option<NumberFor<B>>,
+	/// The last time `display` or `new` has been called.
+	last_update: time::Instant,
+}
+
+impl<B: BlockT> InformantDisplay<B> {
+	/// Builds a new informant display system.
+	pub fn new() -> InformantDisplay<B> {
+		InformantDisplay {
+			last_number: None,
+			last_update: time::Instant::now(),
+		}
+	}
+
+	/// Displays the informant by calling `info!`.
+	pub fn display(&mut self, info: &ClientInfo<B>, net_status: NetworkStatus<B>) {
+		let best_number = info.chain.best_number;
+		let best_hash = info.chain.best_hash;
+		let speed = speed::<B>(best_number, self.last_number, self.last_update);
+		self.last_update = time::Instant::now();
+		self.last_number = Some(best_number);
+
+		let (status, target) = match (net_status.sync_state, net_status.best_seen_block) {
+			(SyncState::Idle, _) => ("Idle".into(), "".into()),
+			(SyncState::Downloading, None) => (format!("Syncing{}", speed), "".into()),
+			(SyncState::Downloading, Some(n)) => (format!("Syncing{}", speed), format!(", target=#{}", n)),
+		};
+
+		info!(
+			target: "substrate",
+			"{}{} ({} peers), best: #{} ({}), finalized #{} ({}), ⬇ {} ⬆ {}",
+			Colour::White.bold().paint(&status),
+			target,
+			Colour::White.bold().paint(format!("{}", net_status.num_connected_peers)),
+			Colour::White.paint(format!("{}", best_number)),
+			best_hash,
+			Colour::White.paint(format!("{}", info.chain.finalized_number)),
+			info.chain.finalized_hash,
+			TransferRateFormat(net_status.average_download_per_sec),
+			TransferRateFormat(net_status.average_upload_per_sec),
+		);
+	}
+}
+
+/// Calculates `(best_number - last_number) / (now - last_update)` and returns a `String`
+/// representing the speed of import.
+fn speed<B: BlockT>(
+	best_number: NumberFor<B>,
+	last_number: Option<NumberFor<B>>,
+	last_update: time::Instant
+) -> String {
+	// Number of milliseconds elapsed since last time.
+	let elapsed_ms = {
+		let elapsed = last_update.elapsed();
+		let since_last_millis = elapsed.as_secs() * 1000;
+		let since_last_subsec_millis = elapsed.subsec_millis() as u64;
+		since_last_millis + since_last_subsec_millis
+	};
+
+	// Number of blocks that have been imported since last time.
+	let diff = match last_number {
+		None => return String::new(),
+		Some(n) => best_number.saturating_sub(n)
+	};
+
+	if let Ok(diff) = TryInto::<u128>::try_into(diff) {
+		// If the number of blocks can be converted to a regular integer, then it's easy: just
+		// do the math and turn it into a `f64`.
+		let speed = diff.saturating_mul(10_000).checked_div(u128::from(elapsed_ms))
+			.map_or(0.0, |s| s as f64) / 10.0;
+		format!(" {:4.1} bps", speed)
+
+	} else {
+		// If the number of blocks can't be converted to a regular integer, then we need a more
+		// algebraic approach and we stay within the realm of integers.
+		let one_thousand = NumberFor::<B>::from(1_000);
+		let elapsed = NumberFor::<B>::from(
+			<u32 as TryFrom<_>>::try_from(elapsed_ms).unwrap_or(u32::max_value())
+		);
+
+		let speed = diff.saturating_mul(one_thousand).checked_div(&elapsed)
+			.unwrap_or_else(Zero::zero);
+		format!(" {} bps", speed)
+	}
+}
+
+/// Contains a number of bytes per second. Implements `fmt::Display` and shows this number of bytes
+/// per second in a nice way.
+struct TransferRateFormat(u64);
+impl fmt::Display for TransferRateFormat {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		// Special case 0.
+		if self.0 == 0 {
+			return write!(f, "0")
+		}
+
+		// Under 0.1 kiB, display plain bytes.
+		if self.0 < 100 {
+			return write!(f, "{} B/s", self.0)
+		}
+
+		// Under 1.0 MiB/sec, display the value in kiB/sec.
+		if self.0 < 1024 * 1024 {
+			return write!(f, "{:.1}kiB/s", self.0 as f64 / 1024.0)
+		}
+
+		write!(f, "{:.1}MiB/s", self.0 as f64 / (1024.0 * 1024.0))
+	}
+}

substrate/client/cli/src/params.rs

@@ -0,0 +1,915 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::traits::{AugmentClap, GetLogFilter};
+
+use std::path::PathBuf;
+use structopt::{StructOpt, clap::{arg_enum, App, AppSettings, SubCommand, Arg}};
+
+pub use crate::execution_strategy::ExecutionStrategy;
+
+/// Auxiliary macro to implement `GetLogFilter` for all types that have the `shared_params` field.
+macro_rules! impl_get_log_filter {
+	( $type:ident ) => {
+		impl $crate::GetLogFilter for $type {
+			fn get_log_filter(&self) -> Option<String> {
+				self.shared_params.get_log_filter()
+			}
+		}
+	}
+}
+
+impl Into<client_api::ExecutionStrategy> for ExecutionStrategy {
+	fn into(self) -> client_api::ExecutionStrategy {
+		match self {
+			ExecutionStrategy::Native => client_api::ExecutionStrategy::NativeWhenPossible,
+			ExecutionStrategy::Wasm => client_api::ExecutionStrategy::AlwaysWasm,
+			ExecutionStrategy::Both => client_api::ExecutionStrategy::Both,
+			ExecutionStrategy::NativeElseWasm => client_api::ExecutionStrategy::NativeElseWasm,
+		}
+	}
+}
+
+arg_enum! {
+	/// How to execute Wasm runtime code
+	#[allow(missing_docs)]
+	#[derive(Debug, Clone)]
+	pub enum WasmExecutionMethod {
+		// Uses an interpreter.
+		Interpreted,
+		// Uses a compiled runtime.
+		Compiled,
+	}
+}
+
+impl WasmExecutionMethod {
+	/// Returns list of variants that are not disabled by feature flags.
+	fn enabled_variants() -> Vec<&'static str> {
+		Self::variants()
+			.iter()
+			.cloned()
+			.filter(|&name| cfg!(feature = "wasmtime") || name != "Compiled")
+			.collect()
+	}
+}
+
+impl Into<service::config::WasmExecutionMethod> for WasmExecutionMethod {
+	fn into(self) -> service::config::WasmExecutionMethod {
+		match self {
+			WasmExecutionMethod::Interpreted => service::config::WasmExecutionMethod::Interpreted,
+			#[cfg(feature = "wasmtime")]
+			WasmExecutionMethod::Compiled => service::config::WasmExecutionMethod::Compiled,
+			#[cfg(not(feature = "wasmtime"))]
+			WasmExecutionMethod::Compiled => panic!(
+				"Substrate must be compiled with \"wasmtime\" feature for compiled Wasm execution"
+			),
+		}
+	}
+}
+
+arg_enum! {
+	/// Whether off-chain workers are enabled.
+	#[allow(missing_docs)]
+	#[derive(Debug, Clone)]
+	pub enum OffchainWorkerEnabled {
+		Always,
+		Never,
+		WhenValidating,
+	}
+}
+
+/// Shared parameters used by all `CoreParams`.
+#[derive(Debug, StructOpt, Clone)]
+pub struct SharedParams {
+	/// Specify the chain specification (one of dev, local or staging).
+	#[structopt(long = "chain", value_name = "CHAIN_SPEC")]
+	pub chain: Option<String>,
+
+	/// Specify the development chain.
+	#[structopt(long = "dev")]
+	pub dev: bool,
+
+	/// Specify custom base path.
+	#[structopt(long = "base-path", short = "d", value_name = "PATH", parse(from_os_str))]
+	pub base_path: Option<PathBuf>,
+
+	/// Sets a custom logging filter.
+	#[structopt(short = "l", long = "log", value_name = "LOG_PATTERN")]
+	pub log: Option<String>,
+}
+
+impl GetLogFilter for SharedParams {
+	fn get_log_filter(&self) -> Option<String> {
+		self.log.clone()
+	}
+}
+
+/// Parameters used to create the network configuration.
+#[derive(Debug, StructOpt, Clone)]
+pub struct NetworkConfigurationParams {
+	/// Specify a list of bootnodes.
+	#[structopt(long = "bootnodes", value_name = "URL")]
+	pub bootnodes: Vec<String>,
+
+	/// Specify a list of reserved node addresses.
+	#[structopt(long = "reserved-nodes", value_name = "URL")]
+	pub reserved_nodes: Vec<String>,
+
+	/// Whether to only allow connections to/from reserved nodes.
+	///
+	/// If you are a validator your node might still connect to other validator
+	/// nodes regardless of whether they are defined as reserved nodes.
+	#[structopt(long = "reserved-only")]
+	pub reserved_only: bool,
+
+	/// Listen on this multiaddress.
+	#[structopt(long = "listen-addr", value_name = "LISTEN_ADDR")]
+	pub listen_addr: Vec<String>,
+
+	/// Specify p2p protocol TCP port.
+	///
+	/// Only used if --listen-addr is not specified.
+	#[structopt(long = "port", value_name = "PORT")]
+	pub port: Option<u16>,
+
+	/// Allow connecting to private IPv4 addresses (as specified in
+	/// [RFC1918](https://tools.ietf.org/html/rfc1918)), unless the address was passed with
+	/// `--reserved-nodes` or `--bootnodes`.
+	#[structopt(long = "no-private-ipv4")]
+	pub no_private_ipv4: bool,
+
+	/// Specify the number of outgoing connections we're trying to maintain.
+	#[structopt(long = "out-peers", value_name = "COUNT", default_value = "25")]
+	pub out_peers: u32,
+
+	/// Specify the maximum number of incoming connections we're accepting.
+	#[structopt(long = "in-peers", value_name = "COUNT", default_value = "25")]
+	pub in_peers: u32,
+
+	/// Disable mDNS discovery.
+	///
+	/// By default, the network will use mDNS to discover other nodes on the
+	/// local network. This disables it. Automatically implied when using --dev.
+	#[structopt(long = "no-mdns")]
+	pub no_mdns: bool,
+
+	/// Maximum number of peers to ask the same blocks in parallel.
+	///
+	/// This allows downlading announced blocks from multiple peers. Decrease to save
+	/// traffic and risk increased latency.
+	#[structopt(long = "max-parallel-downloads", value_name = "COUNT", default_value = "5")]
+	pub max_parallel_downloads: u32,
+
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub node_key_params: NodeKeyParams
+}
+
+arg_enum! {
+	#[allow(missing_docs)]
+	#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+	pub enum NodeKeyType {
+		Ed25519
+	}
+}
+
+/// Parameters used to create the `NodeKeyConfig`, which determines the keypair
+/// used for libp2p networking.
+#[derive(Debug, StructOpt, Clone)]
+pub struct NodeKeyParams {
+	/// The secret key to use for libp2p networking.
+	///
+	/// The value is a string that is parsed according to the choice of
+	/// `--node-key-type` as follows:
+	///
+	///   `ed25519`:
+	///   The value is parsed as a hex-encoded Ed25519 32 bytes secret key,
+	///   i.e. 64 hex characters.
+	///
+	/// The value of this option takes precedence over `--node-key-file`.
+	///
+	/// WARNING: Secrets provided as command-line arguments are easily exposed.
+	/// Use of this option should be limited to development and testing. To use
+	/// an externally managed secret key, use `--node-key-file` instead.
+	#[structopt(long = "node-key", value_name = "KEY")]
+	pub node_key: Option<String>,
+
+	/// The type of secret key to use for libp2p networking.
+	///
+	/// The secret key of the node is obtained as follows:
+	///
+	///   * If the `--node-key` option is given, the value is parsed as a secret key
+	///     according to the type. See the documentation for `--node-key`.
+	///
+	///   * If the `--node-key-file` option is given, the secret key is read from the
+	///     specified file. See the documentation for `--node-key-file`.
+	///
+	///   * Otherwise, the secret key is read from a file with a predetermined,
+	///     type-specific name from the chain-specific network config directory
+	///     inside the base directory specified by `--base-dir`. If this file does
+	///     not exist, it is created with a newly generated secret key of the
+	///     chosen type.
+	///
+	/// The node's secret key determines the corresponding public key and hence the
+	/// node's peer ID in the context of libp2p.
+	#[structopt(
+		long = "node-key-type",
+		value_name = "TYPE",
+		possible_values = &NodeKeyType::variants(),
+		case_insensitive = true,
+		default_value = "Ed25519"
+	)]
+	pub node_key_type: NodeKeyType,
+
+	/// The file from which to read the node's secret key to use for libp2p networking.
+	///
+	/// The contents of the file are parsed according to the choice of `--node-key-type`
+	/// as follows:
+	///
+	///   `ed25519`:
+	///   The file must contain an unencoded 32 bytes Ed25519 secret key.
+	///
+	/// If the file does not exist, it is created with a newly generated secret key of
+	/// the chosen type.
+	#[structopt(long = "node-key-file", value_name = "FILE")]
+	pub node_key_file: Option<PathBuf>
+}
+
+/// Parameters used to create the pool configuration.
+#[derive(Debug, StructOpt, Clone)]
+pub struct TransactionPoolParams {
+	/// Maximum number of transactions in the transaction pool.
+	#[structopt(long = "pool-limit", value_name = "COUNT", default_value = "512")]
+	pub pool_limit: usize,
+	/// Maximum number of kilobytes of all transactions stored in the pool.
+	#[structopt(long = "pool-kbytes", value_name = "COUNT", default_value = "10240")]
+	pub pool_kbytes: usize,
+}
+
+/// Execution strategies parameters.
+#[derive(Debug, StructOpt, Clone)]
+pub struct ExecutionStrategies {
+	/// The means of execution used when calling into the runtime while syncing blocks.
+	#[structopt(
+		long = "execution-syncing",
+		value_name = "STRATEGY",
+		possible_values = &ExecutionStrategy::variants(),
+		case_insensitive = true,
+		default_value = "NativeElseWasm"
+	)]
+	pub execution_syncing: ExecutionStrategy,
+
+	/// The means of execution used when calling into the runtime while importing blocks.
+	#[structopt(
+		long = "execution-import-block",
+		value_name = "STRATEGY",
+		possible_values = &ExecutionStrategy::variants(),
+		case_insensitive = true,
+		default_value = "NativeElseWasm"
+	)]
+	pub execution_import_block: ExecutionStrategy,
+
+	/// The means of execution used when calling into the runtime while constructing blocks.
+	#[structopt(
+		long = "execution-block-construction",
+		value_name = "STRATEGY",
+		possible_values = &ExecutionStrategy::variants(),
+		case_insensitive = true,
+		default_value = "Wasm"
+	)]
+	pub execution_block_construction: ExecutionStrategy,
+
+	/// The means of execution used when calling into the runtime while using an off-chain worker.
+	#[structopt(
+		long = "execution-offchain-worker",
+		value_name = "STRATEGY",
+		possible_values = &ExecutionStrategy::variants(),
+		case_insensitive = true,
+		default_value = "Native"
+	)]
+	pub execution_offchain_worker: ExecutionStrategy,
+
+	/// The means of execution used when calling into the runtime while not syncing, importing or constructing blocks.
+	#[structopt(
+		long = "execution-other",
+		value_name = "STRATEGY",
+		possible_values = &ExecutionStrategy::variants(),
+		case_insensitive = true,
+		default_value = "Native"
+	)]
+	pub execution_other: ExecutionStrategy,
+
+	/// The execution strategy that should be used by all execution contexts.
+	#[structopt(
+		long = "execution",
+		value_name = "STRATEGY",
+		possible_values = &ExecutionStrategy::variants(),
+		case_insensitive = true,
+		conflicts_with_all = &[
+			"execution-other",
+			"execution-offchain-worker",
+			"execution-block-construction",
+			"execution-import-block",
+			"execution-syncing",
+		]
+	)]
+	pub execution: Option<ExecutionStrategy>,
+}
+
+/// The `run` command used to run a node.
+#[derive(Debug, StructOpt, Clone)]
+pub struct RunCmd {
+	/// Enable validator mode.
+	///
+	/// The node will be started with the authority role and actively
+	/// participate in any consensus task that it can (e.g. depending on
+	/// availability of local keys).
+	#[structopt(
+		long = "validator",
+		conflicts_with_all = &[ "sentry" ]
+	)]
+	pub validator: bool,
+
+	/// Enable sentry mode.
+	///
+	/// The node will be started with the authority role and participate in
+	/// consensus tasks as an "observer", it will never actively participate
+	/// regardless of whether it could (e.g. keys are available locally). This
+	/// mode is useful as a secure proxy for validators (which would run
+	/// detached from the network), since we want this node to participate in
+	/// the full consensus protocols in order to have all needed consensus data
+	/// available to relay to private nodes.
+	#[structopt(
+		long = "sentry",
+		conflicts_with_all = &[ "validator" ]
+	)]
+	pub sentry: bool,
+
+	/// Disable GRANDPA voter when running in validator mode, otherwise disables the GRANDPA observer.
+	#[structopt(long = "no-grandpa")]
+	pub no_grandpa: bool,
+
+	/// Experimental: Run in light client mode.
+	#[structopt(long = "light")]
+	pub light: bool,
+
+	/// Limit the memory the database cache can use.
+	#[structopt(long = "db-cache", value_name = "MiB")]
+	pub database_cache_size: Option<u32>,
+
+	/// Specify the state cache size.
+	#[structopt(long = "state-cache-size", value_name = "Bytes", default_value = "67108864")]
+	pub state_cache_size: usize,
+
+	/// Listen to all RPC interfaces.
+	///
+	/// Default is local.
+	#[structopt(long = "rpc-external")]
+	pub rpc_external: bool,
+
+	/// Listen to all Websocket interfaces.
+	///
+	/// Default is local.
+	#[structopt(long = "ws-external")]
+	pub ws_external: bool,
+
+	/// Specify HTTP RPC server TCP port.
+	#[structopt(long = "rpc-port", value_name = "PORT")]
+	pub rpc_port: Option<u16>,
+
+	/// Specify WebSockets RPC server TCP port.
+	#[structopt(long = "ws-port", value_name = "PORT")]
+	pub ws_port: Option<u16>,
+
+	/// Maximum number of WS RPC server connections.
+	#[structopt(long = "ws-max-connections", value_name = "COUNT")]
+	pub ws_max_connections: Option<usize>,
+
+	/// Specify browser Origins allowed to access the HTTP & WS RPC servers.
+	///
+	/// A comma-separated list of origins (protocol://domain or special `null`
+	/// value). Value of `all` will disable origin validation. Default is to
+	/// allow localhost, https://polkadot.js.org and
+	/// https://substrate-ui.parity.io origins. When running in --dev mode the
+	/// default is to allow all origins.
+	#[structopt(long = "rpc-cors", value_name = "ORIGINS", parse(try_from_str = parse_cors))]
+	pub rpc_cors: Option<Cors>,
+
+	/// Specify the state pruning mode, a number of blocks to keep or 'archive'.
+	///
+	/// Default is to keep all block states if the node is running as a
+	/// validator (i.e. 'archive'), otherwise state is only kept for the last
+	/// 256 blocks.
+	#[structopt(long = "pruning", value_name = "PRUNING_MODE")]
+	pub pruning: Option<String>,
+
+	/// Force start with unsafe pruning settings.
+	///
+	/// When running as a validator it is highly recommended to disable state
+	/// pruning (i.e. 'archive') which is the default. The node will refuse to
+	/// start as a validator if pruning is enabled unless this option is set.
+	#[structopt(long = "unsafe-pruning")]
+	pub unsafe_pruning: bool,
+
+	/// The human-readable name for this node.
+	///
+	/// The node name will be reported to the telemetry server, if enabled.
+	#[structopt(long = "name", value_name = "NAME")]
+	pub name: Option<String>,
+
+	/// Disable connecting to the Substrate telemetry server.
+	///
+	/// Telemetry is on by default on global chains.
+	#[structopt(long = "no-telemetry")]
+	pub no_telemetry: bool,
+
+	/// The URL of the telemetry server to connect to.
+	///
+	/// This flag can be passed multiple times as a mean to specify multiple
+	/// telemetry endpoints. Verbosity levels range from 0-9, with 0 denoting
+	/// the least verbosity. If no verbosity level is specified the default is
+	/// 0.
+	#[structopt(long = "telemetry-url", value_name = "URL VERBOSITY", parse(try_from_str = parse_telemetry_endpoints))]
+	pub telemetry_endpoints: Vec<(String, u8)>,
+
+	/// Should execute offchain workers on every block.
+	///
+	/// By default it's only enabled for nodes that are authoring new blocks.
+	#[structopt(
+		long = "offchain-worker",
+		value_name = "ENABLED",
+		possible_values = &OffchainWorkerEnabled::variants(),
+		case_insensitive = true,
+		default_value = "WhenValidating"
+	)]
+	pub offchain_worker: OffchainWorkerEnabled,
+
+	/// Method for executing Wasm runtime code.
+	#[structopt(
+		long = "wasm-execution",
+		value_name = "METHOD",
+		possible_values = &WasmExecutionMethod::enabled_variants(),
+		case_insensitive = true,
+		default_value = "Interpreted"
+	)]
+	pub wasm_method: WasmExecutionMethod,
+
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub execution_strategies: ExecutionStrategies,
+
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub shared_params: SharedParams,
+
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub network_config: NetworkConfigurationParams,
+
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub pool_config: TransactionPoolParams,
+
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub keyring: Keyring,
+
+	/// Enable authoring even when offline.
+	#[structopt(long = "force-authoring")]
+	pub force_authoring: bool,
+
+	/// Specify custom keystore path.
+	#[structopt(long = "keystore-path", value_name = "PATH", parse(from_os_str))]
+	pub keystore_path: Option<PathBuf>,
+
+	/// Use interactive shell for entering the password used by the keystore.
+	#[structopt(
+		long = "password-interactive",
+		conflicts_with_all = &[ "password", "password-filename" ]
+	)]
+	pub password_interactive: bool,
+
+	/// Password used by the keystore.
+	#[structopt(
+		long = "password",
+		conflicts_with_all = &[ "password-interactive", "password-filename" ]
+	)]
+	pub password: Option<String>,
+
+	/// File that contains the password used by the keystore.
+	#[structopt(
+		long = "password-filename",
+		value_name = "PATH",
+		parse(from_os_str),
+		conflicts_with_all = &[ "password-interactive", "password" ]
+	)]
+	pub password_filename: Option<PathBuf>
+}
+
+/// Stores all required Cli values for a keyring test account.
+struct KeyringTestAccountCliValues {
+	help: String,
+	conflicts_with: Vec<String>,
+	name: String,
+	variant: keyring::Sr25519Keyring,
+}
+
+lazy_static::lazy_static! {
+	/// The Cli values for all test accounts.
+	static ref TEST_ACCOUNTS_CLI_VALUES: Vec<KeyringTestAccountCliValues> = {
+		keyring::Sr25519Keyring::iter().map(|a| {
+			let help = format!(
+				"Shortcut for `--name {} --validator` with session keys for `{}` added to keystore.",
+				a,
+				a,
+			);
+			let conflicts_with = keyring::Sr25519Keyring::iter()
+				.filter(|b| a != *b)
+				.map(|b| b.to_string().to_lowercase())
+				.chain(std::iter::once("name".to_string()))
+				.collect::<Vec<_>>();
+			let name = a.to_string().to_lowercase();
+
+			KeyringTestAccountCliValues {
+				help,
+				conflicts_with,
+				name,
+				variant: a,
+			}
+		}).collect()
+	};
+}
+
+/// Wrapper for exposing the keyring test accounts into the Cli.
+#[derive(Debug, Clone)]
+pub struct Keyring {
+	pub account: Option<keyring::Sr25519Keyring>,
+}
+
+impl StructOpt for Keyring {
+	fn clap<'a, 'b>() -> App<'a, 'b> {
+		unimplemented!("Should not be called for `TestAccounts`.")
+	}
+
+	fn from_clap(m: &::structopt::clap::ArgMatches) -> Self {
+		Keyring {
+			account: TEST_ACCOUNTS_CLI_VALUES.iter().find(|a| m.is_present(&a.name)).map(|a| a.variant),
+		}
+	}
+}
+
+impl AugmentClap for Keyring {
+	fn augment_clap<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> {
+		TEST_ACCOUNTS_CLI_VALUES.iter().fold(app, |app, a| {
+			let conflicts_with_strs = a.conflicts_with.iter().map(|s| s.as_str()).collect::<Vec<_>>();
+
+			app.arg(
+				Arg::with_name(&a.name)
+					.long(&a.name)
+					.help(&a.help)
+					.conflicts_with_all(&conflicts_with_strs)
+					.takes_value(false)
+			)
+		})
+	}
+}
+
+impl Keyring {
+	fn is_subcommand() -> bool {
+		false
+	}
+}
+
+/// Default to verbosity level 0, if none is provided.
+fn parse_telemetry_endpoints(s: &str) -> Result<(String, u8), Box<dyn std::error::Error>> {
+	let pos = s.find(' ');
+	match pos {
+		None => {
+			Ok((s.to_owned(), 0))
+		},
+		Some(pos_) => {
+			let verbosity = s[pos_ + 1..].parse()?;
+			let url = s[..pos_].parse()?;
+			Ok((url, verbosity))
+		}
+	}
+}
+
+/// CORS setting
+///
+/// The type is introduced to overcome `Option<Option<T>>`
+/// handling of `structopt`.
+#[derive(Clone, Debug)]
+pub enum Cors {
+	/// All hosts allowed
+	All,
+	/// Only hosts on the list are allowed.
+	List(Vec<String>),
+}
+
+impl From<Cors> for Option<Vec<String>> {
+	fn from(cors: Cors) -> Self {
+		match cors {
+			Cors::All => None,
+			Cors::List(list) => Some(list),
+		}
+	}
+}
+
+/// Parse cors origins
+fn parse_cors(s: &str) -> Result<Cors, Box<dyn std::error::Error>> {
+	let mut is_all = false;
+	let mut origins = Vec::new();
+	for part in s.split(',') {
+		match part {
+			"all" | "*" => {
+				is_all = true;
+				break;
+			},
+			other => origins.push(other.to_owned()),
+		}
+	}
+
+	Ok(if is_all { Cors::All } else { Cors::List(origins) })
+}
+
+impl_augment_clap!(RunCmd);
+impl_get_log_filter!(RunCmd);
+
+/// The `build-spec` command used to build a specification.
+#[derive(Debug, StructOpt, Clone)]
+pub struct BuildSpecCmd {
+	/// Force raw genesis storage output.
+	#[structopt(long = "raw")]
+	pub raw: bool,
+
+	/// Disable adding the default bootnode to the specification.
+	///
+	/// By default the `/ip4/127.0.0.1/tcp/30333/p2p/NODE_PEER_ID` bootnode is added to the
+	/// specification when no bootnode exists.
+	#[structopt(long = "disable-default-bootnode")]
+	pub disable_default_bootnode: bool,
+
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub shared_params: SharedParams,
+
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub node_key_params: NodeKeyParams,
+}
+
+impl_get_log_filter!(BuildSpecCmd);
+
+/// The `export-blocks` command used to export blocks.
+#[derive(Debug, StructOpt, Clone)]
+pub struct ExportBlocksCmd {
+	/// Output file name or stdout if unspecified.
+	#[structopt(parse(from_os_str))]
+	pub output: Option<PathBuf>,
+
+	/// Specify starting block number.
+	///
+	/// Default is 1.
+	#[structopt(long = "from", value_name = "BLOCK")]
+	pub from: Option<u32>,
+
+	/// Specify last block number.
+	///
+	/// Default is best block.
+	#[structopt(long = "to", value_name = "BLOCK")]
+	pub to: Option<u32>,
+
+	/// Use JSON output rather than binary.
+	#[structopt(long = "json")]
+	pub json: bool,
+
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub shared_params: SharedParams,
+}
+
+impl_get_log_filter!(ExportBlocksCmd);
+
+/// The `import-blocks` command used to import blocks.
+#[derive(Debug, StructOpt, Clone)]
+pub struct ImportBlocksCmd {
+	/// Input file or stdin if unspecified.
+	#[structopt(parse(from_os_str))]
+	pub input: Option<PathBuf>,
+
+	/// The default number of 64KB pages to ever allocate for Wasm execution.
+	///
+	/// Don't alter this unless you know what you're doing.
+	#[structopt(long = "default-heap-pages", value_name = "COUNT")]
+	pub default_heap_pages: Option<u32>,
+
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub shared_params: SharedParams,
+
+	/// Method for executing Wasm runtime code.
+	#[structopt(
+		long = "wasm-execution",
+		value_name = "METHOD",
+		possible_values = &WasmExecutionMethod::variants(),
+		case_insensitive = true,
+		default_value = "Interpreted"
+	)]
+	pub wasm_method: WasmExecutionMethod,
+
+	/// The means of execution used when calling into the runtime while importing blocks.
+	#[structopt(
+		long = "execution",
+		value_name = "STRATEGY",
+		possible_values = &ExecutionStrategy::variants(),
+		case_insensitive = true,
+		default_value = "NativeElseWasm"
+	)]
+	pub execution: ExecutionStrategy,
+}
+
+impl_get_log_filter!(ImportBlocksCmd);
+
+/// The `revert` command used revert the chain to a previous state.
+#[derive(Debug, StructOpt, Clone)]
+pub struct RevertCmd {
+	/// Number of blocks to revert.
+	#[structopt(default_value = "256")]
+	pub num: u32,
+
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub shared_params: SharedParams,
+}
+
+impl_get_log_filter!(RevertCmd);
+
+/// The `purge-chain` command used to remove the whole chain.
+#[derive(Debug, StructOpt, Clone)]
+pub struct PurgeChainCmd {
+	/// Skip interactive prompt by answering yes automatically.
+	#[structopt(short = "y")]
+	pub yes: bool,
+
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub shared_params: SharedParams,
+}
+
+impl_get_log_filter!(PurgeChainCmd);
+
+/// All core commands that are provided by default.
+///
+/// The core commands are split into multiple subcommands and `Run` is the default subcommand. From
+/// the CLI user perspective, it is not visible that `Run` is a subcommand. So, all parameters of
+/// `Run` are exported as main executable parameters.
+#[derive(Debug, Clone)]
+pub enum CoreParams<CC, RP> {
+	/// Run a node.
+	Run(MergeParameters<RunCmd, RP>),
+
+	/// Build a spec.json file, outputing to stdout.
+	BuildSpec(BuildSpecCmd),
+
+	/// Export blocks to a file.
+	ExportBlocks(ExportBlocksCmd),
+
+	/// Import blocks from file.
+	ImportBlocks(ImportBlocksCmd),
+
+	/// Revert chain to the previous state.
+	Revert(RevertCmd),
+
+	/// Remove the whole chain data.
+	PurgeChain(PurgeChainCmd),
+
+	/// Further custom subcommands.
+	Custom(CC),
+}
+
+impl<CC, RP> StructOpt for CoreParams<CC, RP> where
+	CC: StructOpt + GetLogFilter,
+	RP: StructOpt + AugmentClap
+{
+	fn clap<'a, 'b>() -> App<'a, 'b> {
+		RP::augment_clap(
+			RunCmd::augment_clap(
+				CC::clap().unset_setting(AppSettings::SubcommandRequiredElseHelp)
+			)
+		).subcommand(
+			BuildSpecCmd::augment_clap(SubCommand::with_name("build-spec"))
+				.about("Build a spec.json file, outputting to stdout.")
+		)
+		.subcommand(
+			ExportBlocksCmd::augment_clap(SubCommand::with_name("export-blocks"))
+				.about("Export blocks to a file. This file can only be re-imported \
+						if it is in binary format (not JSON!)."
+					)
+		)
+		.subcommand(
+			ImportBlocksCmd::augment_clap(SubCommand::with_name("import-blocks"))
+				.about("Import blocks from file.")
+		)
+		.subcommand(
+			RevertCmd::augment_clap(SubCommand::with_name("revert"))
+				.about("Revert chain to the previous state.")
+		)
+		.subcommand(
+			PurgeChainCmd::augment_clap(SubCommand::with_name("purge-chain"))
+				.about("Remove the whole chain data.")
+		)
+	}
+
+	fn from_clap(matches: &::structopt::clap::ArgMatches) -> Self {
+		match matches.subcommand() {
+			("build-spec", Some(matches)) =>
+				CoreParams::BuildSpec(BuildSpecCmd::from_clap(matches)),
+			("export-blocks", Some(matches)) =>
+				CoreParams::ExportBlocks(ExportBlocksCmd::from_clap(matches)),
+			("import-blocks", Some(matches)) =>
+				CoreParams::ImportBlocks(ImportBlocksCmd::from_clap(matches)),
+			("revert", Some(matches)) => CoreParams::Revert(RevertCmd::from_clap(matches)),
+			("purge-chain", Some(matches)) =>
+				CoreParams::PurgeChain(PurgeChainCmd::from_clap(matches)),
+			(_, None) => CoreParams::Run(MergeParameters::from_clap(matches)),
+			_ => CoreParams::Custom(CC::from_clap(matches)),
+		}
+	}
+}
+
+impl<CC, RP> GetLogFilter for CoreParams<CC, RP> where CC: GetLogFilter {
+	fn get_log_filter(&self) -> Option<String> {
+		match self {
+			CoreParams::Run(c) => c.left.get_log_filter(),
+			CoreParams::BuildSpec(c) => c.get_log_filter(),
+			CoreParams::ExportBlocks(c) => c.get_log_filter(),
+			CoreParams::ImportBlocks(c) => c.get_log_filter(),
+			CoreParams::PurgeChain(c) => c.get_log_filter(),
+			CoreParams::Revert(c) => c.get_log_filter(),
+			CoreParams::Custom(c) => c.get_log_filter(),
+		}
+	}
+}
+
+/// A special commandline parameter that expands to nothing.
+/// Should be used as custom subcommand/run arguments if no custom values are required.
+#[derive(Clone, Debug, Default)]
+pub struct NoCustom {}
+
+impl StructOpt for NoCustom {
+	fn clap<'a, 'b>() -> App<'a, 'b> {
+		App::new("NoCustom")
+	}
+
+	fn from_clap(_: &::structopt::clap::ArgMatches) -> Self {
+		NoCustom {}
+	}
+}
+
+impl AugmentClap for NoCustom {
+	fn augment_clap<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> {
+		app
+	}
+}
+
+impl GetLogFilter for NoCustom {
+	fn get_log_filter(&self) -> Option<String> {
+		None
+	}
+}
+
+/// Merge all CLI parameters of `L` and `R` into the same level.
+#[derive(Clone, Debug)]
+pub struct MergeParameters<L, R> {
+	/// The left side parameters.
+	pub left: L,
+	/// The right side parameters.
+	pub right: R,
+}
+
+impl<L, R> StructOpt for MergeParameters<L, R> where L: StructOpt + AugmentClap, R: StructOpt {
+	fn clap<'a, 'b>() -> App<'a, 'b> {
+		L::augment_clap(R::clap())
+	}
+
+	fn from_clap(matches: &::structopt::clap::ArgMatches) -> Self {
+		MergeParameters {
+			left: L::from_clap(matches),
+			right: R::from_clap(matches),
+		}
+	}
+}

substrate/client/cli/src/traits.rs

@@ -0,0 +1,44 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use structopt::{StructOpt, clap::App};
+
+/// Something that can augment a clap app with further parameters.
+/// `derive(StructOpt)` is implementing this function by default, so a macro `impl_augment_clap!`
+/// is provided to simplify the implementation of this trait.
+pub trait AugmentClap: StructOpt {
+	/// Augment the given clap `App` with further parameters.
+	fn augment_clap<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b>;
+}
+
+/// Macro for implementing the `AugmentClap` trait.
+/// This requires that the given type uses `derive(StructOpt)`!
+#[macro_export]
+macro_rules! impl_augment_clap {
+	( $type:ident ) => {
+		impl $crate::AugmentClap for $type {
+			fn augment_clap<'a, 'b>(app: $crate::App<'a, 'b>) -> $crate::App<'a, 'b> {
+				$type::augment_clap(app)
+			}
+		}
+	}
+}
+
+/// Returns the log filter given by the user as commandline argument.
+pub trait GetLogFilter {
+	/// Returns the set log filter.
+	fn get_log_filter(&self) -> Option<String>;
+}

substrate/client/consensus/aura/Cargo.toml

@@ -0,0 +1,41 @@
+[package]
+name = "substrate-consensus-aura"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Aura consensus algorithm for substrate"
+edition = "2018"
+
+[dependencies]
+app-crypto = { package = "substrate-application-crypto", path = "../../../primitives/application-crypto" }
+aura_primitives = { package = "substrate-consensus-aura-primitives", path = "../../../primitives/consensus/aura" }
+block-builder-api = { package = "substrate-block-builder-runtime-api", path = "../../../primitives/block-builder/runtime-api" }
+client = { package = "substrate-client", path = "../../" }
+client-api = { package = "substrate-client-api", path = "../../api" }
+codec = { package = "parity-scale-codec", version = "1.0.0" }
+consensus_common = { package = "substrate-consensus-common", path = "../../../primitives/consensus/common" }
+derive_more = "0.15.0"
+futures-preview = { version = "0.3.0-alpha.19", features = ["compat"] }
+futures-timer = "0.4.0"
+futures01 = { package = "futures", version = "0.1" }
+inherents = { package = "substrate-inherents", path = "../../../primitives/inherents" }
+keystore = { package = "substrate-keystore", path = "../../keystore" }
+log = "0.4.8"
+parking_lot = "0.9.0"
+primitives = { package = "substrate-primitives", path = "../../../primitives/core" }
+runtime_io = { package = "sr-io", path = "../../../primitives/sr-io" }
+runtime_version = { package = "sr-version", path = "../../../primitives/sr-version" }
+slots = { package = "substrate-consensus-slots", path = "../slots" }
+sr-api = { path = "../../../primitives/sr-api" }
+sr-primitives = {  path = "../../../primitives/sr-primitives" }
+paint-aura = { path = "../../../paint/aura" }
+substrate-telemetry = { path = "../../telemetry" }
+
+[dev-dependencies]
+keyring = { package = "substrate-keyring", path = "../../../primitives/keyring" }
+substrate-executor = { path = "../../executor" }
+network = { package = "substrate-network", path = "../../network", features = ["test-helpers"]}
+service = { package = "substrate-service", path = "../../service" }
+test-client = { package = "substrate-test-runtime-client", path = "../../../test/utils/runtime/client" }
+tokio = "0.1.22"
+env_logger = "0.7.0"
+tempfile = "3.1.0"

substrate/client/consensus/aura/src/digest.rs

@@ -0,0 +1,65 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Aura (Authority-Round) digests
+//!
+//! This implements the digests for AuRa, to allow the private
+//! `CompatibleDigestItem` trait to appear in public interfaces.
+
+use primitives::Pair;
+use aura_primitives::AURA_ENGINE_ID;
+use sr_primitives::generic::{DigestItem, OpaqueDigestItemId};
+use codec::{Encode, Codec};
+use std::fmt::Debug;
+
+type Signature<P> = <P as Pair>::Signature;
+
+/// A digest item which is usable with aura consensus.
+pub trait CompatibleDigestItem<P: Pair>: Sized {
+	/// Construct a digest item which contains a signature on the hash.
+	fn aura_seal(signature: Signature<P>) -> Self;
+
+	/// If this item is an Aura seal, return the signature.
+	fn as_aura_seal(&self) -> Option<Signature<P>>;
+
+	/// Construct a digest item which contains the slot number
+	fn aura_pre_digest(slot_num: u64) -> Self;
+
+	/// If this item is an AuRa pre-digest, return the slot number
+	fn as_aura_pre_digest(&self) -> Option<u64>;
+}
+
+impl<P, Hash> CompatibleDigestItem<P> for DigestItem<Hash> where
+	P: Pair,
+	Signature<P>: Codec,
+	Hash: Debug + Send + Sync + Eq + Clone + Codec + 'static
+{
+	fn aura_seal(signature: Signature<P>) -> Self {
+		DigestItem::Seal(AURA_ENGINE_ID, signature.encode())
+	}
+
+	fn as_aura_seal(&self) -> Option<Signature<P>> {
+		self.try_to(OpaqueDigestItemId::Seal(&AURA_ENGINE_ID))
+	}
+
+	fn aura_pre_digest(slot_num: u64) -> Self {
+		DigestItem::PreRuntime(AURA_ENGINE_ID, slot_num.encode())
+	}
+
+	fn as_aura_pre_digest(&self) -> Option<u64> {
+		self.try_to(OpaqueDigestItemId::PreRuntime(&AURA_ENGINE_ID))
+	}
+}

substrate/client/consensus/aura/src/lib.rs

@@ -0,0 +1,891 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Aura (Authority-round) consensus in substrate.
+//!
+//! Aura works by having a list of authorities A who are expected to roughly
+//! agree on the current time. Time is divided up into discrete slots of t
+//! seconds each. For each slot s, the author of that slot is A[s % |A|].
+//!
+//! The author is allowed to issue one block but not more during that slot,
+//! and it will be built upon the longest valid chain that has been seen.
+//!
+//! Blocks from future steps will be either deferred or rejected depending on how
+//! far in the future they are.
+//!
+//! NOTE: Aura itself is designed to be generic over the crypto used.
+#![forbid(missing_docs, unsafe_code)]
+use std::{sync::Arc, time::Duration, thread, marker::PhantomData, hash::Hash, fmt::Debug, pin::Pin};
+
+use codec::{Encode, Decode, Codec};
+use consensus_common::{self, BlockImport, Environment, Proposer,
+	ForkChoiceStrategy, BlockImportParams, BlockOrigin, Error as ConsensusError,
+	SelectChain,
+};
+use consensus_common::import_queue::{
+	Verifier, BasicQueue, BoxBlockImport, BoxJustificationImport, BoxFinalityProofImport,
+};
+use client_api::{ error::Result as CResult, backend::AuxStore };
+use client::{
+	blockchain::ProvideCache, BlockOf,
+	well_known_cache_keys::{self, Id as CacheKeyId},
+};
+
+use block_builder_api::BlockBuilder as BlockBuilderApi;
+
+use sr_primitives::{generic::{BlockId, OpaqueDigestItemId}, Justification};
+use sr_primitives::traits::{Block as BlockT, Header, DigestItemFor, ProvideRuntimeApi, Zero, Member};
+
+use primitives::crypto::Pair;
+use inherents::{InherentDataProviders, InherentData};
+
+use futures::prelude::*;
+use parking_lot::Mutex;
+use log::{debug, info, trace};
+
+use paint_aura::{
+	InherentType as AuraInherent, AuraInherentData,
+	timestamp::{TimestampInherentData, InherentType as TimestampInherent, InherentError as TIError}
+};
+use substrate_telemetry::{telemetry, CONSENSUS_TRACE, CONSENSUS_DEBUG, CONSENSUS_INFO};
+
+use slots::{CheckedHeader, SlotData, SlotWorker, SlotInfo, SlotCompatible};
+use slots::check_equivocation;
+
+use keystore::KeyStorePtr;
+
+use sr_api::ApiExt;
+
+pub use aura_primitives::*;
+pub use consensus_common::SyncOracle;
+pub use digest::CompatibleDigestItem;
+
+mod digest;
+
+type AuthorityId<P> = <P as Pair>::Public;
+
+/// A slot duration. Create with `get_or_compute`.
+#[derive(Clone, Copy, Debug, Encode, Decode, Hash, PartialOrd, Ord, PartialEq, Eq)]
+pub struct SlotDuration(slots::SlotDuration<u64>);
+
+impl SlotDuration {
+	/// Either fetch the slot duration from disk or compute it from the genesis
+	/// state.
+	pub fn get_or_compute<A, B, C>(client: &C) -> CResult<Self>
+	where
+		A: Codec,
+		B: BlockT,
+		C: AuxStore + ProvideRuntimeApi,
+		C::Api: AuraApi<B, A, Error = client::error::Error>,
+	{
+		slots::SlotDuration::get_or_compute(client, |a, b| a.slot_duration(b)).map(Self)
+	}
+
+	/// Get the slot duration in milliseconds.
+	pub fn get(&self) -> u64 {
+		self.0.get()
+	}
+}
+
+/// Get slot author for given block along with authorities.
+fn slot_author<P: Pair>(slot_num: u64, authorities: &[AuthorityId<P>]) -> Option<&AuthorityId<P>> {
+	if authorities.is_empty() { return None }
+
+	let idx = slot_num % (authorities.len() as u64);
+	assert!(
+		idx <= usize::max_value() as u64,
+		"It is impossible to have a vector with length beyond the address space; qed",
+	);
+
+	let current_author = authorities.get(idx as usize)
+		.expect("authorities not empty; index constrained to list length;\
+				this is a valid index; qed");
+
+	Some(current_author)
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+struct AuraSlotCompatible;
+
+impl SlotCompatible for AuraSlotCompatible {
+	fn extract_timestamp_and_slot(
+		&self,
+		data: &InherentData
+	) -> Result<(TimestampInherent, AuraInherent, std::time::Duration), consensus_common::Error> {
+		data.timestamp_inherent_data()
+			.and_then(|t| data.aura_inherent_data().map(|a| (t, a)))
+			.map_err(Into::into)
+			.map_err(consensus_common::Error::InherentData)
+			.map(|(x, y)| (x, y, Default::default()))
+	}
+}
+
+/// Start the aura worker. The returned future should be run in a futures executor.
+pub fn start_aura<B, C, SC, E, I, P, SO, Error, H>(
+	slot_duration: SlotDuration,
+	client: Arc<C>,
+	select_chain: SC,
+	block_import: I,
+	env: E,
+	sync_oracle: SO,
+	inherent_data_providers: InherentDataProviders,
+	force_authoring: bool,
+	keystore: KeyStorePtr,
+) -> Result<impl futures01::Future<Item = (), Error = ()>, consensus_common::Error> where
+	B: BlockT<Header=H>,
+	C: ProvideRuntimeApi + BlockOf + ProvideCache<B> + AuxStore + Send + Sync,
+	C::Api: AuraApi<B, AuthorityId<P>>,
+	SC: SelectChain<B>,
+	E: Environment<B, Error=Error> + Send + Sync + 'static,
+	E::Proposer: Proposer<B, Error=Error>,
+	<E::Proposer as Proposer<B>>::Create: Unpin + Send,
+	P: Pair + Send + Sync,
+	P::Public: Hash + Member + Encode + Decode,
+	P::Signature: Hash + Member + Encode + Decode,
+	H: Header<Hash=B::Hash>,
+	I: BlockImport<B> + Send + Sync + 'static,
+	Error: ::std::error::Error + Send + From<::consensus_common::Error> + From<I::Error> + 'static,
+	SO: SyncOracle + Send + Sync + Clone,
+{
+	let worker = AuraWorker {
+		client: client.clone(),
+		block_import: Arc::new(Mutex::new(block_import)),
+		env,
+		keystore,
+		sync_oracle: sync_oracle.clone(),
+		force_authoring,
+		_key_type: PhantomData::<P>,
+	};
+	register_aura_inherent_data_provider(
+		&inherent_data_providers,
+		slot_duration.0.slot_duration()
+	)?;
+	Ok(slots::start_slot_worker::<_, _, _, _, _, AuraSlotCompatible>(
+		slot_duration.0,
+		select_chain,
+		worker,
+		sync_oracle,
+		inherent_data_providers,
+		AuraSlotCompatible,
+	).map(|()| Ok::<(), ()>(())).compat())
+}
+
+struct AuraWorker<C, E, I, P, SO> {
+	client: Arc<C>,
+	block_import: Arc<Mutex<I>>,
+	env: E,
+	keystore: KeyStorePtr,
+	sync_oracle: SO,
+	force_authoring: bool,
+	_key_type: PhantomData<P>,
+}
+
+impl<H, B, C, E, I, P, Error, SO> slots::SimpleSlotWorker<B> for AuraWorker<C, E, I, P, SO> where
+	B: BlockT<Header=H>,
+	C: ProvideRuntimeApi + BlockOf + ProvideCache<B> + Sync,
+	C::Api: AuraApi<B, AuthorityId<P>>,
+	E: Environment<B, Error=Error>,
+	E::Proposer: Proposer<B, Error=Error>,
+	<E::Proposer as Proposer<B>>::Create: Unpin + Send,
+	H: Header<Hash=B::Hash>,
+	I: BlockImport<B> + Send + Sync + 'static,
+	P: Pair + Send + Sync,
+	P::Public: Member + Encode + Decode + Hash,
+	P::Signature: Member + Encode + Decode + Hash + Debug,
+	SO: SyncOracle + Send + Clone,
+	Error: ::std::error::Error + Send + From<::consensus_common::Error> + From<I::Error> + 'static,
+{
+	type EpochData = Vec<AuthorityId<P>>;
+	type Claim = P;
+	type SyncOracle = SO;
+	type Proposer = E::Proposer;
+	type BlockImport = I;
+
+	fn logging_target(&self) -> &'static str {
+		"aura"
+	}
+
+	fn block_import(&self) -> Arc<Mutex<Self::BlockImport>> {
+		self.block_import.clone()
+	}
+
+	fn epoch_data(&self, header: &B::Header, _slot_number: u64) -> Result<Self::EpochData, consensus_common::Error> {
+		authorities(self.client.as_ref(), &BlockId::Hash(header.hash()))
+	}
+
+	fn authorities_len(&self, epoch_data: &Self::EpochData) -> usize {
+		epoch_data.len()
+	}
+
+	fn claim_slot(
+		&self,
+		_header: &B::Header,
+		slot_number: u64,
+		epoch_data: &Self::EpochData,
+	) -> Option<Self::Claim> {
+		let expected_author = slot_author::<P>(slot_number, epoch_data);
+
+		expected_author.and_then(|p| {
+			self.keystore.read()
+				.key_pair_by_type::<P>(&p, app_crypto::key_types::AURA).ok()
+		})
+	}
+
+	fn pre_digest_data(&self, slot_number: u64, _claim: &Self::Claim) -> Vec<sr_primitives::DigestItem<B::Hash>> {
+		vec![
+			<DigestItemFor<B> as CompatibleDigestItem<P>>::aura_pre_digest(slot_number),
+		]
+	}
+
+	fn block_import_params(&self) -> Box<dyn Fn(
+		B::Header,
+		&B::Hash,
+		Vec<B::Extrinsic>,
+		Self::Claim,
+	) -> consensus_common::BlockImportParams<B> + Send> {
+		Box::new(|header, header_hash, body, pair| {
+			// sign the pre-sealed hash of the block and then
+			// add it to a digest item.
+			let signature = pair.sign(header_hash.as_ref());
+			let signature_digest_item = <DigestItemFor<B> as CompatibleDigestItem<P>>::aura_seal(signature);
+
+			BlockImportParams {
+				origin: BlockOrigin::Own,
+				header,
+				justification: None,
+				post_digests: vec![signature_digest_item],
+				body: Some(body),
+				finalized: false,
+				auxiliary: Vec::new(),
+				fork_choice: ForkChoiceStrategy::LongestChain,
+				allow_missing_state: false,
+			}
+		})
+	}
+
+	fn force_authoring(&self) -> bool {
+		self.force_authoring
+	}
+
+	fn sync_oracle(&mut self) -> &mut Self::SyncOracle {
+		&mut self.sync_oracle
+	}
+
+	fn proposer(&mut self, block: &B::Header) -> Result<Self::Proposer, consensus_common::Error> {
+		self.env.init(block).map_err(|e| {
+			consensus_common::Error::ClientImport(format!("{:?}", e)).into()
+		})
+	}
+}
+
+impl<H, B: BlockT, C, E, I, P, Error, SO> SlotWorker<B> for AuraWorker<C, E, I, P, SO> where
+	B: BlockT<Header=H>,
+	C: ProvideRuntimeApi + BlockOf + ProvideCache<B> + Sync + Send,
+	C::Api: AuraApi<B, AuthorityId<P>>,
+	E: Environment<B, Error=Error> + Send + Sync,
+	E::Proposer: Proposer<B, Error=Error>,
+	<E::Proposer as Proposer<B>>::Create: Unpin + Send + 'static,
+	H: Header<Hash=B::Hash>,
+	I: BlockImport<B> + Send + Sync + 'static,
+	P: Pair + Send + Sync,
+	P::Public: Member + Encode + Decode + Hash,
+	P::Signature: Member + Encode + Decode + Hash + Debug,
+	SO: SyncOracle + Send + Sync + Clone,
+	Error: ::std::error::Error + Send + From<::consensus_common::Error> + From<I::Error> + 'static,
+{
+	type OnSlot = Pin<Box<dyn Future<Output = Result<(), consensus_common::Error>> + Send>>;
+
+	fn on_slot(&mut self, chain_head: B::Header, slot_info: SlotInfo) -> Self::OnSlot {
+		<Self as slots::SimpleSlotWorker<B>>::on_slot(self, chain_head, slot_info)
+	}
+}
+
+fn aura_err<B: BlockT>(error: Error<B>) -> Error<B> {
+	debug!(target: "aura", "{}", error);
+	error
+}
+
+#[derive(derive_more::Display)]
+enum Error<B: BlockT> {
+	#[display(fmt = "Multiple Aura pre-runtime headers")]
+	MultipleHeaders,
+	#[display(fmt = "No Aura pre-runtime digest found")]
+	NoDigestFound,
+	#[display(fmt = "Header {:?} is unsealed", _0)]
+	HeaderUnsealed(B::Hash),
+	#[display(fmt = "Header {:?} has a bad seal", _0)]
+	HeaderBadSeal(B::Hash),
+	#[display(fmt = "Slot Author not found")]
+	SlotAuthorNotFound,
+	#[display(fmt = "Bad signature on {:?}", _0)]
+	BadSignature(B::Hash),
+	#[display(fmt = "Rejecting block too far in future")]
+	TooFarInFuture,
+	Client(client::error::Error),
+	DataProvider(String),
+	Runtime(String),
+}
+
+fn find_pre_digest<B: BlockT, P: Pair>(header: &B::Header) -> Result<u64, Error<B>>
+	where DigestItemFor<B>: CompatibleDigestItem<P>,
+		P::Signature: Decode,
+		P::Public: Encode + Decode + PartialEq + Clone,
+{
+	let mut pre_digest: Option<u64> = None;
+	for log in header.digest().logs() {
+		trace!(target: "aura", "Checking log {:?}", log);
+		match (log.as_aura_pre_digest(), pre_digest.is_some()) {
+			(Some(_), true) => Err(aura_err(Error::MultipleHeaders))?,
+			(None, _) => trace!(target: "aura", "Ignoring digest not meant for us"),
+			(s, false) => pre_digest = s,
+		}
+	}
+	pre_digest.ok_or_else(|| aura_err(Error::NoDigestFound))
+}
+
+/// check a header has been signed by the right key. If the slot is too far in the future, an error will be returned.
+/// if it's successful, returns the pre-header and the digest item containing the seal.
+///
+/// This digest item will always return `Some` when used with `as_aura_seal`.
+//
+// FIXME #1018 needs misbehavior types. The `transaction_pool` parameter will be
+// used to submit such misbehavior reports.
+fn check_header<C, B: BlockT, P: Pair, T>(
+	client: &C,
+	slot_now: u64,
+	mut header: B::Header,
+	hash: B::Hash,
+	authorities: &[AuthorityId<P>],
+	_transaction_pool: Option<&T>,
+) -> Result<CheckedHeader<B::Header, (u64, DigestItemFor<B>)>, Error<B>> where
+	DigestItemFor<B>: CompatibleDigestItem<P>,
+	P::Signature: Decode,
+	C: client_api::backend::AuxStore,
+	P::Public: Encode + Decode + PartialEq + Clone,
+	T: Send + Sync + 'static,
+{
+	let seal = match header.digest_mut().pop() {
+		Some(x) => x,
+		None => return Err(Error::HeaderUnsealed(hash)),
+	};
+
+	let sig = seal.as_aura_seal().ok_or_else(|| {
+		aura_err(Error::HeaderBadSeal(hash))
+	})?;
+
+	let slot_num = find_pre_digest::<B, _>(&header)?;
+
+	if slot_num > slot_now {
+		header.digest_mut().push(seal);
+		Ok(CheckedHeader::Deferred(header, slot_num))
+	} else {
+		// check the signature is valid under the expected authority and
+		// chain state.
+		let expected_author = match slot_author::<P>(slot_num, &authorities) {
+			None => return Err(Error::SlotAuthorNotFound),
+			Some(author) => author,
+		};
+
+		let pre_hash = header.hash();
+
+		if P::verify(&sig, pre_hash.as_ref(), expected_author) {
+			if let Some(equivocation_proof) = check_equivocation(
+				client,
+				slot_now,
+				slot_num,
+				&header,
+				expected_author,
+			).map_err(Error::Client)? {
+				info!(
+					"Slot author is equivocating at slot {} with headers {:?} and {:?}",
+					slot_num,
+					equivocation_proof.fst_header().hash(),
+					equivocation_proof.snd_header().hash(),
+				);
+			}
+
+			Ok(CheckedHeader::Checked(header, (slot_num, seal)))
+		} else {
+			Err(Error::BadSignature(hash))
+		}
+	}
+}
+
+/// A verifier for Aura blocks.
+pub struct AuraVerifier<C, P, T> {
+	client: Arc<C>,
+	phantom: PhantomData<P>,
+	inherent_data_providers: inherents::InherentDataProviders,
+	transaction_pool: Option<Arc<T>>,
+}
+
+impl<C, P, T> AuraVerifier<C, P, T>
+	where P: Send + Sync + 'static
+{
+	fn check_inherents<B: BlockT>(
+		&self,
+		block: B,
+		block_id: BlockId<B>,
+		inherent_data: InherentData,
+		timestamp_now: u64,
+	) -> Result<(), Error<B>>
+		where C: ProvideRuntimeApi, C::Api: BlockBuilderApi<B, Error = client::error::Error>
+	{
+		const MAX_TIMESTAMP_DRIFT_SECS: u64 = 60;
+
+		let inherent_res = self.client.runtime_api().check_inherents(
+			&block_id,
+			block,
+			inherent_data,
+		).map_err(Error::Client)?;
+
+		if !inherent_res.ok() {
+			inherent_res
+				.into_errors()
+				.try_for_each(|(i, e)| match TIError::try_from(&i, &e) {
+					Some(TIError::ValidAtTimestamp(timestamp)) => {
+						// halt import until timestamp is valid.
+						// reject when too far ahead.
+						if timestamp > timestamp_now + MAX_TIMESTAMP_DRIFT_SECS {
+							return Err(Error::TooFarInFuture);
+						}
+
+						let diff = timestamp.saturating_sub(timestamp_now);
+						info!(
+							target: "aura",
+							"halting for block {} seconds in the future",
+							diff
+						);
+						telemetry!(CONSENSUS_INFO; "aura.halting_for_future_block";
+							"diff" => ?diff
+						);
+						thread::sleep(Duration::from_secs(diff));
+						Ok(())
+					},
+					Some(TIError::Other(e)) => Err(Error::Runtime(e.into())),
+					None => Err(Error::DataProvider(
+						self.inherent_data_providers.error_to_string(&i, &e)
+					)),
+				})
+		} else {
+			Ok(())
+		}
+	}
+}
+
+#[forbid(deprecated)]
+impl<B: BlockT, C, P, T> Verifier<B> for AuraVerifier<C, P, T> where
+	C: ProvideRuntimeApi + Send + Sync + client_api::backend::AuxStore + ProvideCache<B> + BlockOf,
+	C::Api: BlockBuilderApi<B> + AuraApi<B, AuthorityId<P>> + ApiExt<B, Error = client::error::Error>,
+	DigestItemFor<B>: CompatibleDigestItem<P>,
+	P: Pair + Send + Sync + 'static,
+	P::Public: Send + Sync + Hash + Eq + Clone + Decode + Encode + Debug + 'static,
+	P::Signature: Encode + Decode,
+	T: Send + Sync + 'static,
+{
+	fn verify(
+		&mut self,
+		origin: BlockOrigin,
+		header: B::Header,
+		justification: Option<Justification>,
+		mut body: Option<Vec<B::Extrinsic>>,
+	) -> Result<(BlockImportParams<B>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
+		let mut inherent_data = self.inherent_data_providers
+			.create_inherent_data()
+			.map_err(|e| e.into_string())?;
+		let (timestamp_now, slot_now, _) = AuraSlotCompatible.extract_timestamp_and_slot(&inherent_data)
+			.map_err(|e| format!("Could not extract timestamp and slot: {:?}", e))?;
+		let hash = header.hash();
+		let parent_hash = *header.parent_hash();
+		let authorities = authorities(self.client.as_ref(), &BlockId::Hash(parent_hash))
+			.map_err(|e| format!("Could not fetch authorities at {:?}: {:?}", parent_hash, e))?;
+
+		// we add one to allow for some small drift.
+		// FIXME #1019 in the future, alter this queue to allow deferring of
+		// headers
+		let checked_header = check_header::<C, B, P, T>(
+			&self.client,
+			slot_now + 1,
+			header,
+			hash,
+			&authorities[..],
+			self.transaction_pool.as_ref().map(|x| &**x),
+		).map_err(|e| e.to_string())?;
+		match checked_header {
+			CheckedHeader::Checked(pre_header, (slot_num, seal)) => {
+				// if the body is passed through, we need to use the runtime
+				// to check that the internally-set timestamp in the inherents
+				// actually matches the slot set in the seal.
+				if let Some(inner_body) = body.take() {
+					inherent_data.aura_replace_inherent_data(slot_num);
+					let block = B::new(pre_header.clone(), inner_body);
+
+					// skip the inherents verification if the runtime API is old.
+					if self.client
+						.runtime_api()
+						.has_api_with::<dyn BlockBuilderApi<B, Error = ()>, _>(
+							&BlockId::Hash(parent_hash),
+							|v| v >= 2,
+						)
+						.map_err(|e| format!("{:?}", e))?
+					{
+						self.check_inherents(
+							block.clone(),
+							BlockId::Hash(parent_hash),
+							inherent_data,
+							timestamp_now,
+						).map_err(|e| e.to_string())?;
+					}
+
+					let (_, inner_body) = block.deconstruct();
+					body = Some(inner_body);
+				}
+
+				trace!(target: "aura", "Checked {:?}; importing.", pre_header);
+				telemetry!(CONSENSUS_TRACE; "aura.checked_and_importing"; "pre_header" => ?pre_header);
+
+				// Look for an authorities-change log.
+				let maybe_keys = pre_header.digest()
+					.logs()
+					.iter()
+					.filter_map(|l| l.try_to::<ConsensusLog<AuthorityId<P>>>(
+						OpaqueDigestItemId::Consensus(&AURA_ENGINE_ID)
+					))
+					.find_map(|l| match l {
+						ConsensusLog::AuthoritiesChange(a) => Some(
+							vec![(well_known_cache_keys::AUTHORITIES, a.encode())]
+						),
+						_ => None,
+					});
+
+				let block_import_params = BlockImportParams {
+					origin,
+					header: pre_header,
+					post_digests: vec![seal],
+					body,
+					finalized: false,
+					justification,
+					auxiliary: Vec::new(),
+					fork_choice: ForkChoiceStrategy::LongestChain,
+					allow_missing_state: false,
+				};
+
+				Ok((block_import_params, maybe_keys))
+			}
+			CheckedHeader::Deferred(a, b) => {
+				debug!(target: "aura", "Checking {:?} failed; {:?}, {:?}.", hash, a, b);
+				telemetry!(CONSENSUS_DEBUG; "aura.header_too_far_in_future";
+					"hash" => ?hash, "a" => ?a, "b" => ?b
+				);
+				Err(format!("Header {:?} rejected: too far in the future", hash))
+			}
+		}
+	}
+}
+
+fn initialize_authorities_cache<A, B, C>(client: &C) -> Result<(), ConsensusError> where
+	A: Codec,
+	B: BlockT,
+	C: ProvideRuntimeApi + BlockOf + ProvideCache<B>,
+	C::Api: AuraApi<B, A>,
+{
+	// no cache => no initialization
+	let cache = match client.cache() {
+		Some(cache) => cache,
+		None => return Ok(()),
+	};
+
+	// check if we already have initialized the cache
+	let genesis_id = BlockId::Number(Zero::zero());
+	let genesis_authorities: Option<Vec<A>> = cache
+		.get_at(&well_known_cache_keys::AUTHORITIES, &genesis_id)
+		.and_then(|(_, _, v)| Decode::decode(&mut &v[..]).ok());
+	if genesis_authorities.is_some() {
+		return Ok(());
+	}
+
+	let map_err = |error| consensus_common::Error::from(consensus_common::Error::ClientImport(
+		format!(
+			"Error initializing authorities cache: {}",
+			error,
+		)));
+	let genesis_authorities = authorities(client, &genesis_id)?;
+	cache.initialize(&well_known_cache_keys::AUTHORITIES, genesis_authorities.encode())
+		.map_err(map_err)?;
+
+	Ok(())
+}
+
+#[allow(deprecated)]
+fn authorities<A, B, C>(client: &C, at: &BlockId<B>) -> Result<Vec<A>, ConsensusError> where
+	A: Codec,
+	B: BlockT,
+	C: ProvideRuntimeApi + BlockOf + ProvideCache<B>,
+	C::Api: AuraApi<B, A>,
+{
+	client
+		.cache()
+		.and_then(|cache| cache
+			.get_at(&well_known_cache_keys::AUTHORITIES, at)
+			.and_then(|(_, _, v)| Decode::decode(&mut &v[..]).ok())
+		)
+		.or_else(|| AuraApi::authorities(&*client.runtime_api(), at).ok())
+		.ok_or_else(|| consensus_common::Error::InvalidAuthoritiesSet.into())
+}
+
+/// The Aura import queue type.
+pub type AuraImportQueue<B> = BasicQueue<B>;
+
+/// Register the aura inherent data provider, if not registered already.
+fn register_aura_inherent_data_provider(
+	inherent_data_providers: &InherentDataProviders,
+	slot_duration: u64,
+) -> Result<(), consensus_common::Error> {
+	if !inherent_data_providers.has_provider(&paint_aura::INHERENT_IDENTIFIER) {
+		inherent_data_providers
+			.register_provider(paint_aura::InherentDataProvider::new(slot_duration))
+			.map_err(Into::into)
+			.map_err(consensus_common::Error::InherentData)
+	} else {
+		Ok(())
+	}
+}
+
+/// Start an import queue for the Aura consensus algorithm.
+pub fn import_queue<B, C, P, T>(
+	slot_duration: SlotDuration,
+	block_import: BoxBlockImport<B>,
+	justification_import: Option<BoxJustificationImport<B>>,
+	finality_proof_import: Option<BoxFinalityProofImport<B>>,
+	client: Arc<C>,
+	inherent_data_providers: InherentDataProviders,
+	transaction_pool: Option<Arc<T>>,
+) -> Result<AuraImportQueue<B>, consensus_common::Error> where
+	B: BlockT,
+	C: 'static + ProvideRuntimeApi + BlockOf + ProvideCache<B> + Send + Sync + AuxStore,
+	C::Api: BlockBuilderApi<B> + AuraApi<B, AuthorityId<P>> + ApiExt<B, Error = client::error::Error>,
+	DigestItemFor<B>: CompatibleDigestItem<P>,
+	P: Pair + Send + Sync + 'static,
+	P::Public: Clone + Eq + Send + Sync + Hash + Debug + Encode + Decode,
+	P::Signature: Encode + Decode,
+	T: Send + Sync + 'static,
+{
+	register_aura_inherent_data_provider(&inherent_data_providers, slot_duration.get())?;
+	initialize_authorities_cache(&*client)?;
+
+	let verifier = AuraVerifier {
+		client: client.clone(),
+		inherent_data_providers,
+		phantom: PhantomData,
+		transaction_pool,
+	};
+	Ok(BasicQueue::new(
+		verifier,
+		block_import,
+		justification_import,
+		finality_proof_import,
+	))
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use consensus_common::NoNetwork as DummyOracle;
+	use network::test::*;
+	use network::test::{Block as TestBlock, PeersClient, PeersFullClient};
+	use sr_primitives::traits::{Block as BlockT, DigestFor};
+	use network::config::ProtocolConfig;
+	use parking_lot::Mutex;
+	use tokio::runtime::current_thread;
+	use keyring::sr25519::Keyring;
+	use client::BlockchainEvents;
+	use test_client;
+	use aura_primitives::sr25519::AuthorityPair;
+
+	type Error = client::error::Error;
+
+	type TestClient = client::Client<
+		test_client::Backend,
+		test_client::Executor,
+		TestBlock,
+		test_client::runtime::RuntimeApi
+	>;
+
+	struct DummyFactory(Arc<TestClient>);
+	struct DummyProposer(u64, Arc<TestClient>);
+
+	impl Environment<TestBlock> for DummyFactory {
+		type Proposer = DummyProposer;
+		type Error = Error;
+
+		fn init(&mut self, parent_header: &<TestBlock as BlockT>::Header)
+			-> Result<DummyProposer, Error>
+		{
+			Ok(DummyProposer(parent_header.number + 1, self.0.clone()))
+		}
+	}
+
+	impl Proposer<TestBlock> for DummyProposer {
+		type Error = Error;
+		type Create = future::Ready<Result<TestBlock, Error>>;
+
+		fn propose(
+			&mut self,
+			_: InherentData,
+			digests: DigestFor<TestBlock>,
+			_: Duration,
+		) -> Self::Create {
+			let r = self.1.new_block(digests).unwrap().bake().map_err(|e| e.into());
+			future::ready(r)
+		}
+	}
+
+	const SLOT_DURATION: u64 = 1000;
+
+	pub struct AuraTestNet {
+		peers: Vec<Peer<(), DummySpecialization>>,
+	}
+
+	impl TestNetFactory for AuraTestNet {
+		type Specialization = DummySpecialization;
+		type Verifier = AuraVerifier<PeersFullClient, AuthorityPair, ()>;
+		type PeerData = ();
+
+		/// Create new test network with peers and given config.
+		fn from_config(_config: &ProtocolConfig) -> Self {
+			AuraTestNet {
+				peers: Vec::new(),
+			}
+		}
+
+		fn make_verifier(&self, client: PeersClient, _cfg: &ProtocolConfig, _peer_data: &())
+			-> Self::Verifier
+		{
+			match client {
+				PeersClient::Full(client, _) => {
+					let slot_duration = SlotDuration::get_or_compute(&*client)
+						.expect("slot duration available");
+					let inherent_data_providers = InherentDataProviders::new();
+					register_aura_inherent_data_provider(
+						&inherent_data_providers,
+						slot_duration.get()
+					).expect("Registers aura inherent data provider");
+
+					assert_eq!(slot_duration.get(), SLOT_DURATION);
+					AuraVerifier {
+						client,
+						inherent_data_providers,
+						transaction_pool: Default::default(),
+						phantom: Default::default(),
+					}
+				},
+				PeersClient::Light(_, _) => unreachable!("No (yet) tests for light client + Aura"),
+			}
+		}
+
+		fn peer(&mut self, i: usize) -> &mut Peer<Self::PeerData, DummySpecialization> {
+			&mut self.peers[i]
+		}
+
+		fn peers(&self) -> &Vec<Peer<Self::PeerData, DummySpecialization>> {
+			&self.peers
+		}
+
+		fn mut_peers<F: FnOnce(&mut Vec<Peer<Self::PeerData, DummySpecialization>>)>(&mut self, closure: F) {
+			closure(&mut self.peers);
+		}
+	}
+
+	#[test]
+	#[allow(deprecated)]
+	fn authoring_blocks() {
+		let _ = env_logger::try_init();
+		let net = AuraTestNet::new(3);
+
+		let peers = &[
+			(0, Keyring::Alice),
+			(1, Keyring::Bob),
+			(2, Keyring::Charlie),
+		];
+
+		let net = Arc::new(Mutex::new(net));
+		let mut import_notifications = Vec::new();
+
+		let mut runtime = current_thread::Runtime::new().unwrap();
+		let mut keystore_paths = Vec::new();
+		for (peer_id, key) in peers {
+			let mut net = net.lock();
+			let peer = net.peer(*peer_id);
+			let client = peer.client().as_full().expect("full clients are created").clone();
+			let select_chain = peer.select_chain().expect("full client has a select chain");
+			let keystore_path = tempfile::tempdir().expect("Creates keystore path");
+			let keystore = keystore::Store::open(keystore_path.path(), None).expect("Creates keystore.");
+
+			keystore.write().insert_ephemeral_from_seed::<AuthorityPair>(&key.to_seed())
+				.expect("Creates authority key");
+			keystore_paths.push(keystore_path);
+
+			let environ = DummyFactory(client.clone());
+			import_notifications.push(
+				client.import_notification_stream()
+					.take_while(|n| future::ready(!(n.origin != BlockOrigin::Own && n.header.number() < &5)))
+					.for_each(move |_| future::ready(()))
+			);
+
+			let slot_duration = SlotDuration::get_or_compute(&*client)
+				.expect("slot duration available");
+
+			let inherent_data_providers = InherentDataProviders::new();
+			register_aura_inherent_data_provider(
+				&inherent_data_providers, slot_duration.get()
+			).expect("Registers aura inherent data provider");
+
+			let aura = start_aura::<_, _, _, _, _, AuthorityPair, _, _, _>(
+				slot_duration,
+				client.clone(),
+				select_chain,
+				client,
+				environ,
+				DummyOracle,
+				inherent_data_providers,
+				false,
+				keystore,
+			).expect("Starts aura");
+
+			runtime.spawn(aura);
+		}
+
+		runtime.spawn(futures01::future::poll_fn(move || {
+			net.lock().poll();
+			Ok::<_, ()>(futures01::Async::NotReady::<()>)
+		}));
+
+		runtime.block_on(future::join_all(import_notifications)
+			.map(|_| Ok::<(), ()>(())).compat()).unwrap();
+	}
+
+	#[test]
+	fn authorities_call_works() {
+		let client = test_client::new();
+
+		assert_eq!(client.info().chain.best_number, 0);
+		assert_eq!(authorities(&client, &BlockId::Number(0)).unwrap(), vec![
+			Keyring::Alice.public().into(),
+			Keyring::Bob.public().into(),
+			Keyring::Charlie.public().into()
+		]);
+	}
+}

substrate/client/consensus/babe/Cargo.toml

@@ -0,0 +1,55 @@
+[package]
+name = "substrate-consensus-babe"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "BABE consensus algorithm for substrate"
+edition = "2018"
+
+[dependencies]
+codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] }
+babe_primitives = { package = "substrate-consensus-babe-primitives", path = "../../../primitives/consensus/babe" }
+primitives = { package = "substrate-primitives", path = "../../../primitives/core" }
+app-crypto = { package = "substrate-application-crypto", path = "../../../primitives/application-crypto" }
+num-bigint = "0.2.3"
+num-rational = "0.2.2"
+num-traits = "0.2.8"
+runtime-version = { package = "sr-version", path = "../../../primitives/sr-version" }
+runtime-io = { package = "sr-io", path = "../../../primitives/sr-io" }
+inherents = { package = "substrate-inherents", path = "../../../primitives/inherents" }
+substrate-telemetry = { path = "../../telemetry" }
+keystore = { package = "substrate-keystore", path = "../../keystore" }
+paint-babe = { path = "../../../paint/babe" }
+client-api = { package = "substrate-client-api", path = "../../api" }
+client = { package = "substrate-client", path = "../../" }
+sr-api = { path = "../../../primitives/sr-api" }
+block-builder-api = { package = "substrate-block-builder-runtime-api", path = "../../../primitives/block-builder/runtime-api" }
+header-metadata = { package = "substrate-header-metadata", path = "../../header-metadata" }
+consensus-common = { package = "substrate-consensus-common", path = "../../../primitives/consensus/common" }
+uncles = { package = "substrate-consensus-uncles", path = "../uncles" }
+slots = { package = "substrate-consensus-slots", path = "../slots"  }
+sr-primitives = {  path = "../../../primitives/sr-primitives" }
+fork-tree = { path = "../../../utils/fork-tree" }
+futures-preview = { version = "0.3.0-alpha.19", features = ["compat"] }
+futures01 = { package = "futures", version = "0.1" }
+futures-timer = "0.4.0"
+parking_lot = "0.9.0"
+log = "0.4.8"
+schnorrkel = { version = "0.8.5", features = ["preaudit_deprecated"] }
+rand = "0.7.2"
+merlin = "1.2.1"
+pdqselect = "0.1.0"
+derive_more = "0.15.0"
+
+[dev-dependencies]
+keyring = { package = "substrate-keyring", path = "../../../primitives/keyring" }
+substrate-executor = { path = "../../executor" }
+network = { package = "substrate-network", path = "../../network", features = ["test-helpers"]}
+service = { package = "substrate-service", path = "../../service" }
+test-client = { package = "substrate-test-runtime-client", path = "../../../test/utils/runtime/client" }
+block-builder = { package = "substrate-block-builder", path = "../../block-builder" }
+tokio = "0.1.22"
+env_logger = "0.7.0"
+tempfile = "3.1.0"
+
+[features]
+test-helpers = []

substrate/client/consensus/babe/src/authorship.rs

@@ -0,0 +1,214 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! BABE authority selection and slot claiming.
+
+use merlin::Transcript;
+use babe_primitives::{AuthorityId, BabeAuthorityWeight, BABE_ENGINE_ID, BABE_VRF_PREFIX};
+use babe_primitives::{Epoch, SlotNumber, AuthorityPair, BabePreDigest, BabeConfiguration};
+use primitives::{U256, blake2_256};
+use codec::Encode;
+use schnorrkel::vrf::VRFInOut;
+use primitives::Pair;
+use keystore::KeyStorePtr;
+
+/// Calculates the primary selection threshold for a given authority, taking
+/// into account `c` (`1 - c` represents the probability of a slot being empty).
+pub(super) fn calculate_primary_threshold(
+	c: (u64, u64),
+	authorities: &[(AuthorityId, BabeAuthorityWeight)],
+	authority_index: usize,
+) -> u128 {
+	use num_bigint::BigUint;
+	use num_rational::BigRational;
+	use num_traits::{cast::ToPrimitive, identities::One};
+
+	let c = c.0 as f64 / c.1 as f64;
+
+	let theta =
+		authorities[authority_index].1 as f64 /
+		authorities.iter().map(|(_, weight)| weight).sum::<u64>() as f64;
+
+	let calc = || {
+		let p = BigRational::from_float(1f64 - (1f64 - c).powf(theta))?;
+		let numer = p.numer().to_biguint()?;
+		let denom = p.denom().to_biguint()?;
+		((BigUint::one() << 128) * numer / denom).to_u128()
+	};
+
+	calc().unwrap_or(u128::max_value())
+}
+
+/// Returns true if the given VRF output is lower than the given threshold,
+/// false otherwise.
+pub(super) fn check_primary_threshold(inout: &VRFInOut, threshold: u128) -> bool {
+	u128::from_le_bytes(inout.make_bytes::<[u8; 16]>(BABE_VRF_PREFIX)) < threshold
+}
+
+/// Get the expected secondary author for the given slot and with given
+/// authorities. This should always assign the slot to some authority unless the
+/// authorities list is empty.
+pub(super) fn secondary_slot_author(
+	slot_number: u64,
+	authorities: &[(AuthorityId, BabeAuthorityWeight)],
+	randomness: [u8; 32],
+) -> Option<&AuthorityId> {
+	if authorities.is_empty() {
+		return None;
+	}
+
+	let rand = U256::from((randomness, slot_number).using_encoded(blake2_256));
+
+	let authorities_len = U256::from(authorities.len());
+	let idx = rand % authorities_len;
+
+	let expected_author = authorities.get(idx.as_u32() as usize)
+		.expect("authorities not empty; index constrained to list length; \
+				this is a valid index; qed");
+
+	Some(&expected_author.0)
+}
+
+#[allow(deprecated)]
+pub(super) fn make_transcript(
+	randomness: &[u8],
+	slot_number: u64,
+	epoch: u64,
+) -> Transcript {
+	let mut transcript = Transcript::new(&BABE_ENGINE_ID);
+	transcript.commit_bytes(b"slot number", &slot_number.to_le_bytes());
+	transcript.commit_bytes(b"current epoch", &epoch.to_le_bytes());
+	transcript.commit_bytes(b"chain randomness", randomness);
+	transcript
+}
+
+
+/// Claim a secondary slot if it is our turn to propose, returning the
+/// pre-digest to use when authoring the block, or `None` if it is not our turn
+/// to propose.
+fn claim_secondary_slot(
+	slot_number: SlotNumber,
+	authorities: &[(AuthorityId, BabeAuthorityWeight)],
+	keystore: &KeyStorePtr,
+	randomness: [u8; 32],
+) -> Option<(BabePreDigest, AuthorityPair)> {
+	if authorities.is_empty() {
+		return None;
+	}
+
+	let expected_author = super::authorship::secondary_slot_author(
+		slot_number,
+		authorities,
+		randomness,
+	)?;
+
+	let keystore = keystore.read();
+
+	for (pair, authority_index) in authorities.iter()
+		.enumerate()
+		.flat_map(|(i, a)| {
+			keystore.key_pair::<AuthorityPair>(&a.0).ok().map(|kp| (kp, i))
+		})
+	{
+		if pair.public() == *expected_author {
+			let pre_digest = BabePreDigest::Secondary {
+				slot_number,
+				authority_index: authority_index as u32,
+			};
+
+			return Some((pre_digest, pair));
+		}
+	}
+
+	None
+}
+
+/// Tries to claim the given slot number. This method starts by trying to claim
+/// a primary VRF based slot. If we are not able to claim it, then if we have
+/// secondary slots enabled for the given epoch, we will fallback to trying to
+/// claim a secondary slot.
+pub(super) fn claim_slot(
+	slot_number: SlotNumber,
+	epoch: &Epoch,
+	config: &BabeConfiguration,
+	keystore: &KeyStorePtr,
+) -> Option<(BabePreDigest, AuthorityPair)> {
+	claim_primary_slot(slot_number, epoch, config.c, keystore)
+		.or_else(|| {
+			if config.secondary_slots {
+				claim_secondary_slot(
+					slot_number,
+					&epoch.authorities,
+					keystore,
+					epoch.randomness,
+				)
+			} else {
+				None
+			}
+		})
+}
+
+fn get_keypair(q: &AuthorityPair) -> &schnorrkel::Keypair {
+	use primitives::crypto::IsWrappedBy;
+	primitives::sr25519::Pair::from_ref(q).as_ref()
+}
+
+/// Claim a primary slot if it is our turn.  Returns `None` if it is not our turn.
+/// This hashes the slot number, epoch, genesis hash, and chain randomness into
+/// the VRF.  If the VRF produces a value less than `threshold`, it is our turn,
+/// so it returns `Some(_)`. Otherwise, it returns `None`.
+fn claim_primary_slot(
+	slot_number: SlotNumber,
+	epoch: &Epoch,
+	c: (u64, u64),
+	keystore: &KeyStorePtr,
+) -> Option<(BabePreDigest, AuthorityPair)> {
+	let Epoch { authorities, randomness, epoch_index, .. } = epoch;
+	let keystore = keystore.read();
+
+	for (pair, authority_index) in authorities.iter()
+		.enumerate()
+		.flat_map(|(i, a)| {
+			keystore.key_pair::<AuthorityPair>(&a.0).ok().map(|kp| (kp, i))
+		})
+	{
+		let transcript = super::authorship::make_transcript(randomness, slot_number, *epoch_index);
+
+		// Compute the threshold we will use.
+		//
+		// We already checked that authorities contains `key.public()`, so it can't
+		// be empty.  Therefore, this division in `calculate_threshold` is safe.
+		let threshold = super::authorship::calculate_primary_threshold(c, authorities, authority_index);
+
+		let pre_digest = get_keypair(&pair)
+			.vrf_sign_after_check(transcript, |inout| super::authorship::check_primary_threshold(inout, threshold))
+			.map(|s| {
+				BabePreDigest::Primary {
+					slot_number,
+					vrf_output: s.0.to_output(),
+					vrf_proof: s.1,
+					authority_index: authority_index as u32,
+				}
+			});
+
+		// early exit on first successful claim
+		if let Some(pre_digest) = pre_digest {
+			return Some((pre_digest, pair));
+		}
+	}
+
+	None
+}

substrate/client/consensus/babe/src/aux_schema.rs

@@ -0,0 +1,103 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Schema for BABE epoch changes in the aux-db.
+
+use log::info;
+use codec::{Decode, Encode};
+
+use client_api::{
+	backend::AuxStore,
+	error::{Result as ClientResult, Error as ClientError},
+};
+use sr_primitives::traits::Block as BlockT;
+use babe_primitives::BabeBlockWeight;
+
+use super::{epoch_changes::EpochChangesFor, SharedEpochChanges};
+
+const BABE_EPOCH_CHANGES: &[u8] = b"babe_epoch_changes";
+
+fn block_weight_key<H: Encode>(block_hash: H) -> Vec<u8> {
+	(b"block_weight", block_hash).encode()
+}
+
+fn load_decode<B, T>(backend: &B, key: &[u8]) -> ClientResult<Option<T>>
+	where
+		B: AuxStore,
+		T: Decode,
+{
+	let corrupt = |e: codec::Error| {
+		ClientError::Backend(format!("BABE DB is corrupted. Decode error: {}", e.what()))
+	};
+	match backend.get_aux(key)? {
+		None => Ok(None),
+		Some(t) => T::decode(&mut &t[..]).map(Some).map_err(corrupt)
+	}
+}
+
+/// Load or initialize persistent epoch change data from backend.
+pub(crate) fn load_epoch_changes<Block: BlockT, B: AuxStore>(
+	backend: &B,
+) -> ClientResult<SharedEpochChanges<Block>> {
+	let epoch_changes = load_decode::<_, EpochChangesFor<Block>>(backend, BABE_EPOCH_CHANGES)?
+		.map(Into::into)
+		.unwrap_or_else(|| {
+			info!(target: "babe",
+				"Creating empty BABE epoch changes on what appears to be first startup."
+			);
+			SharedEpochChanges::new()
+		});
+
+	Ok(epoch_changes)
+}
+
+/// Update the epoch changes on disk after a change.
+pub(crate) fn write_epoch_changes<Block: BlockT, F, R>(
+	epoch_changes: &EpochChangesFor<Block>,
+	write_aux: F,
+) -> R where
+	F: FnOnce(&[(&'static [u8], &[u8])]) -> R,
+{
+	let encoded_epoch_changes = epoch_changes.encode();
+	write_aux(
+		&[(BABE_EPOCH_CHANGES, encoded_epoch_changes.as_slice())],
+	)
+}
+
+/// Write the cumulative chain-weight of a block ot aux storage.
+pub(crate) fn write_block_weight<H: Encode, F, R>(
+	block_hash: H,
+	block_weight: &BabeBlockWeight,
+	write_aux: F,
+) -> R where
+	F: FnOnce(&[(Vec<u8>, &[u8])]) -> R,
+{
+
+	let key = block_weight_key(block_hash);
+	block_weight.using_encoded(|s|
+		write_aux(
+			&[(key, s)],
+		)
+	)
+}
+
+/// Load the cumulative chain-weight associated with a block.
+pub(crate) fn load_block_weight<H: Encode, B: AuxStore>(
+	backend: &B,
+	block_hash: H,
+) -> ClientResult<Option<BabeBlockWeight>> {
+	load_decode(backend, block_weight_key(block_hash).as_slice())
+}

substrate/client/consensus/babe/src/epoch_changes.rs

@@ -0,0 +1,658 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Handling epoch changes in BABE.
+//!
+//! This exposes the `SharedEpochChanges`, which is a wrapper around a
+//! persistent DAG superimposed over the forks of the blockchain.
+
+use std::sync::Arc;
+use babe_primitives::{Epoch, SlotNumber, NextEpochDescriptor};
+use fork_tree::ForkTree;
+use parking_lot::{Mutex, MutexGuard};
+use sr_primitives::traits::{Block as BlockT, NumberFor, One, Zero};
+use codec::{Encode, Decode};
+use client_api::{
+	error::Error as ClientError,
+	utils::is_descendent_of,
+	blockchain::HeaderBackend
+};
+use header_metadata::HeaderMetadata;
+use primitives::H256;
+use std::ops::Add;
+
+/// A builder for `is_descendent_of` functions.
+pub trait IsDescendentOfBuilder<Hash> {
+	/// The error returned by the function.
+	type Error: std::error::Error;
+	/// A function that can tell you if the second parameter is a descendent of
+	/// the first.
+	type IsDescendentOf: Fn(&Hash, &Hash) -> Result<bool, Self::Error>;
+
+	/// Build an `is_descendent_of` function.
+	///
+	/// The `current` parameter can be `Some` with the details a fresh block whose
+	/// details aren't yet stored, but its parent is.
+	///
+	/// The format of `current` when `Some` is `(current, current_parent)`.
+	fn build_is_descendent_of(&self, current: Option<(Hash, Hash)>)
+		-> Self::IsDescendentOf;
+}
+
+/// Produce a descendent query object given the client.
+pub(crate) fn descendent_query<H, Block>(client: &H) -> HeaderBackendDescendentBuilder<&H, Block> {
+	HeaderBackendDescendentBuilder(client, std::marker::PhantomData)
+}
+
+/// Wrapper to get around unconstrained type errors when implementing
+/// `IsDescendentOfBuilder` for header backends.
+pub(crate) struct HeaderBackendDescendentBuilder<H, Block>(H, std::marker::PhantomData<Block>);
+
+// TODO: relying on Hash = H256 is awful.
+// https://github.com/paritytech/substrate/issues/3624
+impl<'a, H, Block> IsDescendentOfBuilder<H256>
+	for HeaderBackendDescendentBuilder<&'a H, Block> where
+	H: HeaderBackend<Block> + HeaderMetadata<Block, Error=ClientError>,
+	Block: BlockT<Hash = H256>,
+{
+	type Error = ClientError;
+	type IsDescendentOf = Box<dyn Fn(&H256, &H256) -> Result<bool, ClientError> + 'a>;
+
+	fn build_is_descendent_of(&self, current: Option<(H256, H256)>)
+		-> Self::IsDescendentOf
+	{
+		Box::new(is_descendent_of(self.0, current))
+	}
+}
+
+/// An unimported genesis epoch.
+pub struct UnimportedGenesis(Epoch);
+
+/// The viable epoch under which a block can be verified.
+///
+/// If this is the first non-genesis block in the chain, then it will
+/// hold an `UnimportedGenesis` epoch.
+pub enum ViableEpoch {
+	Genesis(UnimportedGenesis),
+	Regular(Epoch),
+}
+
+impl From<Epoch> for ViableEpoch {
+	fn from(epoch: Epoch) -> ViableEpoch {
+		ViableEpoch::Regular(epoch)
+	}
+}
+
+impl AsRef<Epoch> for ViableEpoch {
+	fn as_ref(&self) -> &Epoch {
+		match *self {
+			ViableEpoch::Genesis(UnimportedGenesis(ref e)) => e,
+			ViableEpoch::Regular(ref e) => e,
+		}
+	}
+}
+
+impl ViableEpoch {
+	/// Extract the underlying epoch, disregarding the fact that a genesis
+	/// epoch may be unimported.
+	pub fn into_inner(self) -> Epoch {
+		match self {
+			ViableEpoch::Genesis(UnimportedGenesis(e)) => e,
+			ViableEpoch::Regular(e) => e,
+		}
+	}
+
+	/// Increment the epoch, yielding an `IncrementedEpoch` to be imported
+	/// into the fork-tree.
+	pub fn increment(&self, next_descriptor: NextEpochDescriptor) -> IncrementedEpoch {
+		let next = self.as_ref().increment(next_descriptor);
+		let to_persist = match *self {
+			ViableEpoch::Genesis(UnimportedGenesis(ref epoch_0)) =>
+				PersistedEpoch::Genesis(epoch_0.clone(), next),
+			ViableEpoch::Regular(_) => PersistedEpoch::Regular(next),
+		};
+
+		IncrementedEpoch(to_persist)
+	}
+}
+
+/// The datatype encoded on disk.
+// This really shouldn't be public, but the encode/decode derives force it to be.
+#[derive(Clone, Encode, Decode)]
+pub enum PersistedEpoch {
+	// epoch_0, epoch_1,
+	Genesis(Epoch, Epoch),
+	// epoch_n
+	Regular(Epoch),
+}
+
+/// A fresh, incremented epoch to import into the underlying fork-tree.
+///
+/// Create this with `ViableEpoch::increment`.
+#[must_use = "Freshly-incremented epoch must be imported with `EpochChanges::import`"]
+pub struct IncrementedEpoch(PersistedEpoch);
+
+impl AsRef<Epoch> for IncrementedEpoch {
+	fn as_ref(&self) -> &Epoch {
+		match self.0 {
+			PersistedEpoch::Genesis(_, ref epoch_1) => epoch_1,
+			PersistedEpoch::Regular(ref epoch_n) => epoch_n,
+		}
+	}
+}
+
+/// Tree of all epoch changes across all *seen* forks. Data stored in tree is
+/// the hash and block number of the block signaling the epoch change, and the
+/// epoch that was signalled at that block.
+///
+/// BABE special-cases the first epoch, epoch_0, by saying that it starts at
+/// slot number of the first block in the chain. When bootstrapping a chain,
+/// there can be multiple competing block #1s, so we have to ensure that the overlayed
+/// DAG doesn't get confused.
+///
+/// The first block of every epoch should be producing a descriptor for the next
+/// epoch - this is checked in higher-level code. So the first block of epoch_0 contains
+/// a descriptor for epoch_1. We special-case these and bundle them together in the
+/// same DAG entry, pinned to a specific block #1.
+///
+/// Further epochs (epoch_2, ..., epoch_n) each get their own entry.
+#[derive(Clone, Encode, Decode)]
+pub struct EpochChanges<Hash, Number> {
+	inner: ForkTree<Hash, Number, PersistedEpoch>,
+}
+
+// create a fake header hash which hasn't been included in the chain.
+fn fake_head_hash<H: AsRef<[u8]> + AsMut<[u8]> + Clone>(parent_hash: &H) -> H {
+	let mut h = parent_hash.clone();
+	// dirty trick: flip the first bit of the parent hash to create a hash
+	// which has not been in the chain before (assuming a strong hash function).
+	h.as_mut()[0] ^= 0b10000000;
+	h
+}
+
+impl<Hash, Number> EpochChanges<Hash, Number> where
+	Hash: PartialEq + AsRef<[u8]> + AsMut<[u8]> + Copy,
+	Number: Ord + One + Zero + Add<Output=Number> + Copy,
+{
+	/// Create a new epoch-change tracker.
+	fn new() -> Self {
+		EpochChanges { inner: ForkTree::new() }
+	}
+
+	/// Prune out finalized epochs, except for the ancestor of the finalized
+	/// block. The given slot should be the slot number at which the finalized
+	/// block was authored.
+	pub fn prune_finalized<D: IsDescendentOfBuilder<Hash>>(
+		&mut self,
+		descendent_of_builder: D,
+		hash: &Hash,
+		number: Number,
+		slot: SlotNumber,
+	) -> Result<(), fork_tree::Error<D::Error>> {
+		let is_descendent_of = descendent_of_builder
+			.build_is_descendent_of(None);
+
+		let predicate = |epoch: &PersistedEpoch| match *epoch {
+			PersistedEpoch::Genesis(_, ref epoch_1) =>
+				slot >= epoch_1.end_slot(),
+			PersistedEpoch::Regular(ref epoch_n) =>
+				slot >= epoch_n.end_slot(),
+		};
+
+		// prune any epochs which could not be _live_ as of the children of the
+		// finalized block, i.e. re-root the fork tree to the oldest ancestor of
+		// (hash, number) where epoch.end_slot() >= finalized_slot
+		self.inner.prune(
+			hash,
+			&number,
+			&is_descendent_of,
+			&predicate,
+		)?;
+
+		Ok(())
+	}
+
+	/// Finds the epoch for a child of the given block, assuming the given slot number.
+	///
+	/// If the returned epoch is an `UnimportedGenesis` epoch, it should be imported into the
+	/// tree.
+	pub fn epoch_for_child_of<D: IsDescendentOfBuilder<Hash>, G>(
+		&self,
+		descendent_of_builder: D,
+		parent_hash: &Hash,
+		parent_number: Number,
+		slot_number: SlotNumber,
+		make_genesis: G,
+	) -> Result<Option<ViableEpoch>, fork_tree::Error<D::Error>>
+		where G: FnOnce(SlotNumber) -> Epoch
+	{
+		// find_node_where will give you the node in the fork-tree which is an ancestor
+		// of the `parent_hash` by default. if the last epoch was signalled at the parent_hash,
+		// then it won't be returned. we need to create a new fake chain head hash which
+		// "descends" from our parent-hash.
+		let fake_head_hash = fake_head_hash(parent_hash);
+
+		let is_descendent_of = descendent_of_builder
+			.build_is_descendent_of(Some((fake_head_hash, *parent_hash)));
+
+		if parent_number == Zero::zero() {
+			// need to insert the genesis epoch.
+			let genesis_epoch = make_genesis(slot_number);
+			return Ok(Some(ViableEpoch::Genesis(UnimportedGenesis(genesis_epoch))));
+		}
+
+		// We want to find the deepest node in the tree which is an ancestor
+		// of our block and where the start slot of the epoch was before the
+		// slot of our block. The genesis special-case doesn't need to look
+		// at epoch_1 -- all we're doing here is figuring out which node
+		// we need.
+		let predicate = |epoch: &PersistedEpoch| match *epoch {
+			PersistedEpoch::Genesis(ref epoch_0, _) =>
+				epoch_0.start_slot <= slot_number,
+			PersistedEpoch::Regular(ref epoch_n) =>
+				epoch_n.start_slot <= slot_number,
+		};
+
+		self.inner.find_node_where(
+			&fake_head_hash,
+			&(parent_number + One::one()),
+			&is_descendent_of,
+			&predicate,
+		)
+			.map(|n| n.map(|node| ViableEpoch::Regular(match node.data {
+				// Ok, we found our node.
+				// and here we figure out which of the internal epochs
+				// of a genesis node to use based on their start slot.
+				PersistedEpoch::Genesis(ref epoch_0, ref epoch_1) =>
+					if epoch_1.start_slot <= slot_number {
+						epoch_1.clone()
+					} else {
+						epoch_0.clone()
+					},
+				PersistedEpoch::Regular(ref epoch_n) => epoch_n.clone(),
+			})))
+	}
+
+	/// Import a new epoch-change, signalled at the given block.
+	///
+	/// This assumes that the given block is prospective (i.e. has not been
+	/// imported yet), but its parent has. This is why the parent hash needs
+	/// to be provided.
+	pub fn import<D: IsDescendentOfBuilder<Hash>>(
+		&mut self,
+		descendent_of_builder: D,
+		hash: Hash,
+		number: Number,
+		parent_hash: Hash,
+		epoch: IncrementedEpoch,
+	) -> Result<(), fork_tree::Error<D::Error>> {
+		let is_descendent_of = descendent_of_builder
+			.build_is_descendent_of(Some((hash, parent_hash)));
+
+		let res = self.inner.import(
+			hash,
+			number,
+			epoch.0,
+			&is_descendent_of,
+		);
+
+		match res {
+			Ok(_) | Err(fork_tree::Error::Duplicate) => Ok(()),
+			Err(e) => Err(e),
+		}
+	}
+
+	/// Return the inner fork tree, useful for testing purposes.
+	#[cfg(test)]
+	pub fn tree(&self) -> &ForkTree<Hash, Number, PersistedEpoch> {
+		&self.inner
+	}
+}
+
+/// Type alias to produce the epoch-changes tree from a block type.
+pub type EpochChangesFor<Block> = EpochChanges<<Block as BlockT>::Hash, NumberFor<Block>>;
+
+/// A shared epoch changes tree.
+#[derive(Clone)]
+pub struct SharedEpochChanges<Block: BlockT> {
+	inner: Arc<Mutex<EpochChangesFor<Block>>>,
+}
+
+impl<Block: BlockT> SharedEpochChanges<Block> {
+	/// Create a new instance of the `SharedEpochChanges`.
+	pub fn new() -> Self {
+		SharedEpochChanges {
+			inner: Arc::new(Mutex::new(EpochChanges::<_, _>::new()))
+		}
+	}
+
+	/// Lock the shared epoch changes,
+	pub fn lock(&self) -> MutexGuard<EpochChangesFor<Block>> {
+		self.inner.lock()
+	}
+}
+
+impl<Block: BlockT> From<EpochChangesFor<Block>> for SharedEpochChanges<Block> {
+	fn from(epoch_changes: EpochChangesFor<Block>) -> Self {
+		SharedEpochChanges {
+			inner: Arc::new(Mutex::new(epoch_changes))
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[derive(Debug, PartialEq)]
+	pub struct TestError;
+
+	impl std::fmt::Display for TestError {
+		fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+			write!(f, "TestError")
+		}
+	}
+
+	impl std::error::Error for TestError {}
+
+	impl<'a, F: 'a , H: 'a + PartialEq + std::fmt::Debug> IsDescendentOfBuilder<H> for &'a F
+		where F: Fn(&H, &H) -> Result<bool, TestError>
+	{
+		type Error = TestError;
+		type IsDescendentOf = Box<dyn Fn(&H, &H) -> Result<bool, TestError> + 'a>;
+
+		fn build_is_descendent_of(&self, current: Option<(H, H)>)
+			-> Self::IsDescendentOf
+		{
+			let f = *self;
+			Box::new(move |base, head| {
+				let mut head = head;
+
+				if let Some((ref c_head, ref c_parent)) = current {
+					if head == c_head {
+						if base == c_parent {
+							return Ok(true);
+						} else {
+							head = c_parent;
+						}
+					}
+				}
+
+				f(base, head)
+			})
+		}
+	}
+
+	type Hash = [u8; 1];
+
+	#[test]
+	fn genesis_epoch_is_created_but_not_imported() {
+		//
+		// A - B
+		//  \
+		//   — C
+		//
+		let is_descendent_of = |base: &Hash, block: &Hash| -> Result<bool, TestError> {
+			match (base, *block) {
+				(b"A", b) => Ok(b == *b"B" || b == *b"C" || b == *b"D"),
+				(b"B", b) | (b"C", b) => Ok(b == *b"D"),
+				(b"0", _) => Ok(true),
+				_ => Ok(false),
+			}
+		};
+
+		let make_genesis = |slot| Epoch {
+			epoch_index: 0,
+			start_slot: slot,
+			duration: 100,
+			authorities: Vec::new(),
+			randomness: [0; 32],
+		};
+
+		let epoch_changes = EpochChanges::new();
+		let genesis_epoch = epoch_changes.epoch_for_child_of(
+			&is_descendent_of,
+			b"0",
+			0,
+			10101,
+			&make_genesis,
+		).unwrap().unwrap();
+
+		match genesis_epoch {
+			ViableEpoch::Genesis(_) => {},
+			_ => panic!("should be unimported genesis"),
+		};
+		assert_eq!(genesis_epoch.as_ref(), &make_genesis(10101));
+
+		let genesis_epoch_2 = epoch_changes.epoch_for_child_of(
+			&is_descendent_of,
+			b"0",
+			0,
+			10102,
+			&make_genesis,
+		).unwrap().unwrap();
+
+		match genesis_epoch_2 {
+			ViableEpoch::Genesis(_) => {},
+			_ => panic!("should be unimported genesis"),
+		};
+		assert_eq!(genesis_epoch_2.as_ref(), &make_genesis(10102));
+	}
+
+	#[test]
+	fn epoch_changes_between_blocks() {
+		//
+		// A - B
+		//  \
+		//   — C
+		//
+		let is_descendent_of = |base: &Hash, block: &Hash| -> Result<bool, TestError> {
+			match (base, *block) {
+				(b"A", b) => Ok(b == *b"B" || b == *b"C" || b == *b"D"),
+				(b"B", b) | (b"C", b) => Ok(b == *b"D"),
+				(b"0", _) => Ok(true),
+				_ => Ok(false),
+			}
+		};
+
+		let make_genesis = |slot| Epoch {
+			epoch_index: 0,
+			start_slot: slot,
+			duration: 100,
+			authorities: Vec::new(),
+			randomness: [0; 32],
+		};
+
+		let mut epoch_changes = EpochChanges::new();
+		let genesis_epoch = epoch_changes.epoch_for_child_of(
+			&is_descendent_of,
+			b"0",
+			0,
+			100,
+			&make_genesis,
+		).unwrap().unwrap();
+
+		assert_eq!(genesis_epoch.as_ref(), &make_genesis(100));
+
+		let import_epoch_1 = genesis_epoch.increment(NextEpochDescriptor {
+			authorities: Vec::new(),
+			randomness: [1; 32],
+		});
+		let epoch_1 = import_epoch_1.as_ref().clone();
+
+		epoch_changes.import(
+			&is_descendent_of,
+			*b"A",
+			1,
+			*b"0",
+			import_epoch_1,
+		).unwrap();
+		let genesis_epoch = genesis_epoch.into_inner();
+
+		assert!(is_descendent_of(b"0", b"A").unwrap());
+
+		let end_slot = genesis_epoch.end_slot();
+		assert_eq!(end_slot, epoch_1.start_slot);
+
+		{
+			// x is still within the genesis epoch.
+			let x = epoch_changes.epoch_for_child_of(
+				&is_descendent_of,
+				b"A",
+				1,
+				end_slot - 1,
+				&make_genesis,
+			).unwrap().unwrap().into_inner();
+
+			assert_eq!(x, genesis_epoch);
+		}
+
+		{
+			// x is now at the next epoch, because the block is now at the
+			// start slot of epoch 1.
+			let x = epoch_changes.epoch_for_child_of(
+				&is_descendent_of,
+				b"A",
+				1,
+				end_slot,
+				&make_genesis,
+			).unwrap().unwrap().into_inner();
+
+			assert_eq!(x, epoch_1);
+		}
+
+		{
+			// x is now at the next epoch, because the block is now after
+			// start slot of epoch 1.
+			let x = epoch_changes.epoch_for_child_of(
+				&is_descendent_of,
+				b"A",
+				1,
+				epoch_1.end_slot() - 1,
+				&make_genesis,
+			).unwrap().unwrap().into_inner();
+
+			assert_eq!(x, epoch_1);
+		}
+	}
+
+	#[test]
+	fn two_block_ones_dont_conflict() {
+		//     X - Y
+		//   /
+		// 0 - A - B
+		//
+		let is_descendent_of = |base: &Hash, block: &Hash| -> Result<bool, TestError> {
+			match (base, *block) {
+				(b"A", b) => Ok(b == *b"B"),
+				(b"X", b) => Ok(b == *b"Y"),
+				(b"0", _) => Ok(true),
+				_ => Ok(false),
+			}
+		};
+
+		let duration = 100;
+
+		let make_genesis = |slot| Epoch {
+			epoch_index: 0,
+			start_slot: slot,
+			duration,
+			authorities: Vec::new(),
+			randomness: [0; 32],
+		};
+
+		let mut epoch_changes = EpochChanges::new();
+		let next_descriptor = NextEpochDescriptor {
+			authorities: Vec::new(),
+			randomness: [0; 32],
+		};
+
+		// insert genesis epoch for A
+		{
+			let genesis_epoch_a = epoch_changes.epoch_for_child_of(
+				&is_descendent_of,
+				b"0",
+				0,
+				100,
+				&make_genesis,
+			).unwrap().unwrap();
+
+			epoch_changes.import(
+				&is_descendent_of,
+				*b"A",
+				1,
+				*b"0",
+				genesis_epoch_a.increment(next_descriptor.clone()),
+			).unwrap();
+
+		}
+
+		// insert genesis epoch for X
+		{
+			let genesis_epoch_x = epoch_changes.epoch_for_child_of(
+				&is_descendent_of,
+				b"0",
+				0,
+				1000,
+				&make_genesis,
+			).unwrap().unwrap();
+
+			epoch_changes.import(
+				&is_descendent_of,
+				*b"X",
+				1,
+				*b"0",
+				genesis_epoch_x.increment(next_descriptor.clone()),
+			).unwrap();
+		}
+
+		// now check that the genesis epochs for our respective block 1s
+		// respect the chain structure.
+		{
+			let epoch_for_a_child = epoch_changes.epoch_for_child_of(
+				&is_descendent_of,
+				b"A",
+				1,
+				101,
+				&make_genesis,
+			).unwrap().unwrap();
+
+			assert_eq!(epoch_for_a_child.into_inner(), make_genesis(100));
+
+			let epoch_for_x_child = epoch_changes.epoch_for_child_of(
+				&is_descendent_of,
+				b"X",
+				1,
+				1001,
+				&make_genesis,
+			).unwrap().unwrap();
+
+			assert_eq!(epoch_for_x_child.into_inner(), make_genesis(1000));
+
+			let epoch_for_x_child_before_genesis = epoch_changes.epoch_for_child_of(
+				&is_descendent_of,
+				b"X",
+				1,
+				101,
+				&make_genesis,
+			).unwrap();
+
+			// even though there is a genesis epoch at that slot, it's not in
+			// this chain.
+			assert!(epoch_for_x_child_before_genesis.is_none());
+		}
+	}
+}

substrate/client/consensus/babe/src/tests.rs

@@ -0,0 +1,779 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! BABE testsuite
+
+// FIXME #2532: need to allow deprecated until refactor is done
+// https://github.com/paritytech/substrate/issues/2532
+#![allow(deprecated)]
+use super::*;
+use authorship::claim_slot;
+
+use babe_primitives::{AuthorityPair, SlotNumber};
+use block_builder::BlockBuilder;
+use consensus_common::NoNetwork as DummyOracle;
+use consensus_common::import_queue::{
+	BoxBlockImport, BoxJustificationImport, BoxFinalityProofImport,
+};
+use network::test::*;
+use network::test::{Block as TestBlock, PeersClient};
+use network::config::BoxFinalityProofRequestBuilder;
+use sr_primitives::{generic::DigestItem, traits::{Block as BlockT, DigestFor}};
+use network::config::ProtocolConfig;
+use tokio::runtime::current_thread;
+use client_api::BlockchainEvents;
+use test_client;
+use log::debug;
+use std::{time::Duration, cell::RefCell};
+
+type Item = DigestItem<Hash>;
+
+type Error = client::error::Error;
+
+type TestClient = client::Client<
+	test_client::Backend,
+	test_client::Executor,
+	TestBlock,
+	test_client::runtime::RuntimeApi,
+>;
+
+#[derive(Copy, Clone, PartialEq)]
+enum Stage {
+	PreSeal,
+	PostSeal,
+}
+
+type Mutator = Arc<dyn Fn(&mut TestHeader, Stage) + Send + Sync>;
+
+#[derive(Clone)]
+struct DummyFactory {
+	client: Arc<TestClient>,
+	epoch_changes: crate::SharedEpochChanges<TestBlock>,
+	config: Config,
+	mutator: Mutator,
+}
+
+struct DummyProposer {
+	factory: DummyFactory,
+	parent_hash: Hash,
+	parent_number: u64,
+	parent_slot: SlotNumber,
+}
+
+impl Environment<TestBlock> for DummyFactory {
+	type Proposer = DummyProposer;
+	type Error = Error;
+
+	fn init(&mut self, parent_header: &<TestBlock as BlockT>::Header)
+		-> Result<DummyProposer, Error>
+	{
+
+		let parent_slot = crate::find_pre_digest::<TestBlock>(parent_header)
+			.expect("parent header has a pre-digest")
+			.slot_number();
+
+		Ok(DummyProposer {
+			factory: self.clone(),
+			parent_hash: parent_header.hash(),
+			parent_number: *parent_header.number(),
+			parent_slot,
+		})
+	}
+}
+
+impl DummyProposer {
+	fn propose_with(&mut self, pre_digests: DigestFor<TestBlock>)
+		-> future::Ready<Result<TestBlock, Error>>
+	{
+		use codec::Encode;
+		let block_builder = self.factory.client.new_block_at(
+			&BlockId::Hash(self.parent_hash),
+			pre_digests,
+		).unwrap();
+
+		let mut block = match block_builder.bake().map_err(|e| e.into()) {
+			Ok(b) => b,
+			Err(e) => return future::ready(Err(e)),
+		};
+
+		let this_slot = crate::find_pre_digest::<TestBlock>(block.header())
+			.expect("baked block has valid pre-digest")
+			.slot_number();
+
+		// figure out if we should add a consensus digest, since the test runtime
+		// doesn't.
+		let epoch_changes = self.factory.epoch_changes.lock();
+		let epoch = epoch_changes.epoch_for_child_of(
+			descendent_query(&*self.factory.client),
+			&self.parent_hash,
+			self.parent_number,
+			this_slot,
+			|slot| self.factory.config.genesis_epoch(slot),
+		)
+			.expect("client has data to find epoch")
+			.expect("can compute epoch for baked block")
+			.into_inner();
+
+		let first_in_epoch = self.parent_slot < epoch.start_slot;
+		if first_in_epoch {
+			// push a `Consensus` digest signalling next change.
+			// we just reuse the same randomness and authorities as the prior
+			// epoch. this will break when we add light client support, since
+			// that will re-check the randomness logic off-chain.
+			let digest_data = ConsensusLog::NextEpochData(NextEpochDescriptor {
+				authorities: epoch.authorities.clone(),
+				randomness: epoch.randomness.clone(),
+			}).encode();
+			let digest = DigestItem::Consensus(BABE_ENGINE_ID, digest_data);
+			block.header.digest_mut().push(digest)
+		}
+
+		// mutate the block header according to the mutator.
+		(self.factory.mutator)(&mut block.header, Stage::PreSeal);
+
+		future::ready(Ok(block))
+	}
+}
+
+impl Proposer<TestBlock> for DummyProposer {
+	type Error = Error;
+	type Create = future::Ready<Result<TestBlock, Error>>;
+
+	fn propose(
+		&mut self,
+		_: InherentData,
+		pre_digests: DigestFor<TestBlock>,
+		_: Duration,
+	) -> Self::Create {
+		self.propose_with(pre_digests)
+	}
+}
+
+thread_local! {
+	static MUTATOR: RefCell<Mutator> = RefCell::new(Arc::new(|_, _|()));
+}
+
+#[derive(Clone)]
+struct PanickingBlockImport<B>(B);
+
+impl<B: BlockImport<TestBlock>> BlockImport<TestBlock> for PanickingBlockImport<B> {
+	type Error = B::Error;
+
+	fn import_block(
+		&mut self,
+		block: BlockImportParams<TestBlock>,
+		new_cache: HashMap<CacheKeyId, Vec<u8>>,
+	) -> Result<ImportResult, Self::Error> {
+		Ok(self.0.import_block(block, new_cache).expect("importing block failed"))
+	}
+
+	fn check_block(
+		&mut self,
+		block: BlockCheckParams<TestBlock>,
+	) -> Result<ImportResult, Self::Error> {
+		Ok(self.0.check_block(block).expect("checking block failed"))
+	}
+}
+
+pub struct BabeTestNet {
+	peers: Vec<Peer<Option<PeerData>, DummySpecialization>>,
+}
+
+type TestHeader = <TestBlock as BlockT>::Header;
+type TestExtrinsic = <TestBlock as BlockT>::Extrinsic;
+
+pub struct TestVerifier {
+	inner: BabeVerifier<
+		test_client::Backend,
+		test_client::Executor,
+		TestBlock,
+		test_client::runtime::RuntimeApi,
+		PeersFullClient,
+	>,
+	mutator: Mutator,
+}
+
+impl Verifier<TestBlock> for TestVerifier {
+	/// Verify the given data and return the BlockImportParams and an optional
+	/// new set of validators to import. If not, err with an Error-Message
+	/// presented to the User in the logs.
+	fn verify(
+		&mut self,
+		origin: BlockOrigin,
+		mut header: TestHeader,
+		justification: Option<Justification>,
+		body: Option<Vec<TestExtrinsic>>,
+	) -> Result<(BlockImportParams<TestBlock>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
+		// apply post-sealing mutations (i.e. stripping seal, if desired).
+		(self.mutator)(&mut header, Stage::PostSeal);
+		Ok(self.inner.verify(origin, header, justification, body).expect("verification failed!"))
+	}
+}
+
+pub struct PeerData {
+	link: BabeLink<TestBlock>,
+	inherent_data_providers: InherentDataProviders,
+	block_import: Mutex<Option<BoxBlockImport<TestBlock>>>,
+}
+
+impl TestNetFactory for BabeTestNet {
+	type Specialization = DummySpecialization;
+	type Verifier = TestVerifier;
+	type PeerData = Option<PeerData>;
+
+	/// Create new test network with peers and given config.
+	fn from_config(_config: &ProtocolConfig) -> Self {
+		debug!(target: "babe", "Creating test network from config");
+		BabeTestNet {
+			peers: Vec::new(),
+		}
+	}
+
+	fn make_block_import(&self, client: PeersClient)
+		-> (
+			BoxBlockImport<Block>,
+			Option<BoxJustificationImport<Block>>,
+			Option<BoxFinalityProofImport<Block>>,
+			Option<BoxFinalityProofRequestBuilder<Block>>,
+			Option<PeerData>,
+		)
+	{
+		let client = client.as_full().expect("only full clients are tested");
+		let inherent_data_providers = InherentDataProviders::new();
+
+		let config = Config::get_or_compute(&*client).expect("config available");
+		let (block_import, link) = crate::block_import(
+			config,
+			client.clone(),
+			client.clone(),
+			client.clone(),
+		).expect("can initialize block-import");
+
+		let block_import = PanickingBlockImport(block_import);
+
+		let data_block_import = Mutex::new(Some(Box::new(block_import.clone()) as BoxBlockImport<_>));
+		(
+			Box::new(block_import),
+			None,
+			None,
+			None,
+			Some(PeerData { link, inherent_data_providers, block_import: data_block_import }),
+		)
+	}
+
+	fn make_verifier(
+		&self,
+		client: PeersClient,
+		_cfg: &ProtocolConfig,
+		maybe_link: &Option<PeerData>,
+	)
+		-> Self::Verifier
+	{
+		let client = client.as_full().expect("only full clients are used in test");
+		trace!(target: "babe", "Creating a verifier");
+
+		// ensure block import and verifier are linked correctly.
+		let data = maybe_link.as_ref().expect("babe link always provided to verifier instantiation");
+
+		TestVerifier {
+			inner: BabeVerifier {
+				client: client.clone(),
+				api: client,
+				inherent_data_providers: data.inherent_data_providers.clone(),
+				config: data.link.config.clone(),
+				epoch_changes: data.link.epoch_changes.clone(),
+				time_source: data.link.time_source.clone(),
+			},
+			mutator: MUTATOR.with(|m| m.borrow().clone()),
+		}
+	}
+
+	fn peer(&mut self, i: usize) -> &mut Peer<Self::PeerData, DummySpecialization> {
+		trace!(target: "babe", "Retreiving a peer");
+		&mut self.peers[i]
+	}
+
+	fn peers(&self) -> &Vec<Peer<Self::PeerData, DummySpecialization>> {
+		trace!(target: "babe", "Retreiving peers");
+		&self.peers
+	}
+
+	fn mut_peers<F: FnOnce(&mut Vec<Peer<Self::PeerData, DummySpecialization>>)>(
+		&mut self,
+		closure: F,
+	) {
+		closure(&mut self.peers);
+	}
+}
+
+#[test]
+#[should_panic]
+fn rejects_empty_block() {
+	env_logger::try_init().unwrap();
+	let mut net = BabeTestNet::new(3);
+	let block_builder = |builder: BlockBuilder<_, _>| {
+		builder.bake().unwrap()
+	};
+	net.mut_peers(|peer| {
+		peer[0].generate_blocks(1, BlockOrigin::NetworkInitialSync, block_builder);
+	})
+}
+
+fn run_one_test(
+	mutator: impl Fn(&mut TestHeader, Stage) + Send + Sync + 'static,
+) {
+	let _ = env_logger::try_init();
+	let mutator = Arc::new(mutator) as Mutator;
+
+	MUTATOR.with(|m| *m.borrow_mut() = mutator.clone());
+	let net = BabeTestNet::new(3);
+
+	let peers = &[
+		(0, "//Alice"),
+		(1, "//Bob"),
+		(2, "//Charlie"),
+	];
+
+	let net = Arc::new(Mutex::new(net));
+	let mut import_notifications = Vec::new();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut keystore_paths = Vec::new();
+
+	for (peer_id, seed) in peers {
+		let mut net = net.lock();
+		let peer = net.peer(*peer_id);
+		let client = peer.client().as_full().expect("Only full clients are used in tests").clone();
+		let select_chain = peer.select_chain().expect("Full client has select_chain");
+
+		let keystore_path = tempfile::tempdir().expect("Creates keystore path");
+		let keystore = keystore::Store::open(keystore_path.path(), None).expect("Creates keystore");
+		keystore.write().insert_ephemeral_from_seed::<AuthorityPair>(seed).expect("Generates authority key");
+		keystore_paths.push(keystore_path);
+
+		let mut got_own = false;
+		let mut got_other = false;
+
+		let data = peer.data.as_ref().expect("babe link set up during initialization");
+
+		let environ = DummyFactory {
+			client: client.clone(),
+			config: data.link.config.clone(),
+			epoch_changes: data.link.epoch_changes.clone(),
+			mutator: mutator.clone(),
+		};
+
+		import_notifications.push(
+			// run each future until we get one of our own blocks with number higher than 5
+			// that was produced locally.
+			client.import_notification_stream()
+				.take_while(move |n| future::ready(n.header.number() < &5 || {
+					if n.origin == BlockOrigin::Own {
+						got_own = true;
+					} else {
+						got_other = true;
+					}
+
+					// continue until we have at least one block of our own
+					// and one of another peer.
+					!(got_own && got_other)
+				}))
+				.for_each(|_| future::ready(()) )
+		);
+
+
+		runtime.spawn(start_babe(BabeParams {
+			block_import: data.block_import.lock().take().expect("import set up during init"),
+			select_chain,
+			client,
+			env: environ,
+			sync_oracle: DummyOracle,
+			inherent_data_providers: data.inherent_data_providers.clone(),
+			force_authoring: false,
+			babe_link: data.link.clone(),
+			keystore,
+		}).expect("Starts babe"));
+	}
+
+	runtime.spawn(futures01::future::poll_fn(move || {
+		net.lock().poll();
+		Ok::<_, ()>(futures01::Async::NotReady::<()>)
+	}));
+
+	runtime.block_on(future::join_all(import_notifications)
+		.map(|_| Ok::<(), ()>(())).compat()).unwrap();
+}
+
+#[test]
+fn authoring_blocks() {
+	run_one_test(|_, _| ())
+}
+
+#[test]
+#[should_panic]
+fn rejects_missing_inherent_digest() {
+	run_one_test(|header: &mut TestHeader, stage| {
+		let v = std::mem::replace(&mut header.digest_mut().logs, vec![]);
+		header.digest_mut().logs = v.into_iter()
+			.filter(|v| stage == Stage::PostSeal || v.as_babe_pre_digest().is_none())
+			.collect()
+	})
+}
+
+#[test]
+#[should_panic]
+fn rejects_missing_seals() {
+	run_one_test(|header: &mut TestHeader, stage| {
+		let v = std::mem::replace(&mut header.digest_mut().logs, vec![]);
+		header.digest_mut().logs = v.into_iter()
+			.filter(|v| stage == Stage::PreSeal || v.as_babe_seal().is_none())
+			.collect()
+	})
+}
+
+#[test]
+#[should_panic]
+fn rejects_missing_consensus_digests() {
+	run_one_test(|header: &mut TestHeader, stage| {
+		let v = std::mem::replace(&mut header.digest_mut().logs, vec![]);
+		header.digest_mut().logs = v.into_iter()
+			.filter(|v| stage == Stage::PostSeal || v.as_next_epoch_descriptor().is_none())
+			.collect()
+	});
+}
+
+#[test]
+fn wrong_consensus_engine_id_rejected() {
+	let _ = env_logger::try_init();
+	let sig = AuthorityPair::generate().0.sign(b"");
+	let bad_seal: Item = DigestItem::Seal([0; 4], sig.to_vec());
+	assert!(bad_seal.as_babe_pre_digest().is_none());
+	assert!(bad_seal.as_babe_seal().is_none())
+}
+
+#[test]
+fn malformed_pre_digest_rejected() {
+	let _ = env_logger::try_init();
+	let bad_seal: Item = DigestItem::Seal(BABE_ENGINE_ID, [0; 64].to_vec());
+	assert!(bad_seal.as_babe_pre_digest().is_none());
+}
+
+#[test]
+fn sig_is_not_pre_digest() {
+	let _ = env_logger::try_init();
+	let sig = AuthorityPair::generate().0.sign(b"");
+	let bad_seal: Item = DigestItem::Seal(BABE_ENGINE_ID, sig.to_vec());
+	assert!(bad_seal.as_babe_pre_digest().is_none());
+	assert!(bad_seal.as_babe_seal().is_some())
+}
+
+#[test]
+fn can_author_block() {
+	let _ = env_logger::try_init();
+	let keystore_path = tempfile::tempdir().expect("Creates keystore path");
+	let keystore = keystore::Store::open(keystore_path.path(), None).expect("Creates keystore");
+	let pair = keystore.write().insert_ephemeral_from_seed::<AuthorityPair>("//Alice")
+		.expect("Generates authority pair");
+
+	let mut i = 0;
+	let epoch = Epoch {
+		start_slot: 0,
+		authorities: vec![(pair.public(), 1)],
+		randomness: [0; 32],
+		epoch_index: 1,
+		duration: 100,
+	};
+
+	let mut config = crate::BabeConfiguration {
+		slot_duration: 1000,
+		epoch_length: 100,
+		c: (3, 10),
+		genesis_authorities: Vec::new(),
+		randomness: [0; 32],
+		secondary_slots: true,
+	};
+
+	// with secondary slots enabled it should never be empty
+	match claim_slot(i, &epoch, &config, &keystore) {
+		None => i += 1,
+		Some(s) => debug!(target: "babe", "Authored block {:?}", s.0),
+	}
+
+	// otherwise with only vrf-based primary slots we might need to try a couple
+	// of times.
+	config.secondary_slots = false;
+	loop {
+		match claim_slot(i, &epoch, &config, &keystore) {
+			None => i += 1,
+			Some(s) => {
+				debug!(target: "babe", "Authored block {:?}", s.0);
+				break;
+			}
+		}
+	}
+}
+
+// Propose and import a new BABE block on top of the given parent.
+fn propose_and_import_block(
+	parent: &TestHeader,
+	slot_number: Option<SlotNumber>,
+	proposer_factory: &mut DummyFactory,
+	block_import: &mut BoxBlockImport<TestBlock>,
+) -> H256 {
+	let mut proposer = proposer_factory.init(parent).unwrap();
+
+	let slot_number = slot_number.unwrap_or_else(|| {
+		let parent_pre_digest = find_pre_digest::<TestBlock>(parent).unwrap();
+		parent_pre_digest.slot_number() + 1
+	});
+
+	let pre_digest = sr_primitives::generic::Digest {
+		logs: vec![
+			Item::babe_pre_digest(
+				BabePreDigest::Secondary {
+					authority_index: 0,
+					slot_number,
+				},
+			),
+		],
+	};
+
+	let mut block = futures::executor::block_on(proposer.propose_with(pre_digest)).unwrap();
+
+	let seal = {
+		// sign the pre-sealed hash of the block and then
+		// add it to a digest item.
+		let pair = AuthorityPair::from_seed(&[1; 32]);
+		let pre_hash = block.header.hash();
+		let signature = pair.sign(pre_hash.as_ref());
+		Item::babe_seal(signature)
+	};
+
+	let post_hash = {
+		block.header.digest_mut().push(seal.clone());
+		let h = block.header.hash();
+		block.header.digest_mut().pop();
+		h
+	};
+
+	let import_result = block_import.import_block(
+		BlockImportParams {
+			origin: BlockOrigin::Own,
+			header: block.header,
+			justification: None,
+			post_digests: vec![seal],
+			body: Some(block.extrinsics),
+			finalized: false,
+			auxiliary: Vec::new(),
+			fork_choice: ForkChoiceStrategy::LongestChain,
+			allow_missing_state: false,
+		},
+		Default::default(),
+	).unwrap();
+
+	match import_result {
+		ImportResult::Imported(_) => {},
+		_ => panic!("expected block to be imported"),
+	}
+
+	post_hash
+}
+
+#[test]
+fn importing_block_one_sets_genesis_epoch() {
+	let mut net = BabeTestNet::new(1);
+
+	let peer = net.peer(0);
+	let data = peer.data.as_ref().expect("babe link set up during initialization");
+	let client = peer.client().as_full().expect("Only full clients are used in tests").clone();
+
+	let mut proposer_factory = DummyFactory {
+		client: client.clone(),
+		config: data.link.config.clone(),
+		epoch_changes: data.link.epoch_changes.clone(),
+		mutator: Arc::new(|_, _| ()),
+	};
+
+	let mut block_import = data.block_import.lock().take().expect("import set up during init");
+
+	let genesis_header = client.header(&BlockId::Number(0)).unwrap().unwrap();
+
+	let block_hash = propose_and_import_block(
+		&genesis_header,
+		Some(999),
+		&mut proposer_factory,
+		&mut block_import,
+	);
+
+	let genesis_epoch = data.link.config.genesis_epoch(999);
+
+	let epoch_changes = data.link.epoch_changes.lock();
+	let epoch_for_second_block = epoch_changes.epoch_for_child_of(
+		descendent_query(&*client),
+		&block_hash,
+		1,
+		1000,
+		|slot| data.link.config.genesis_epoch(slot),
+	).unwrap().unwrap().into_inner();
+
+	assert_eq!(epoch_for_second_block, genesis_epoch);
+}
+
+#[test]
+fn importing_epoch_change_block_prunes_tree() {
+	use client_api::Finalizer;
+
+	let mut net = BabeTestNet::new(1);
+
+	let peer = net.peer(0);
+	let data = peer.data.as_ref().expect("babe link set up during initialization");
+
+	let client = peer.client().as_full().expect("Only full clients are used in tests").clone();
+	let mut block_import = data.block_import.lock().take().expect("import set up during init");
+	let epoch_changes = data.link.epoch_changes.clone();
+
+	let mut proposer_factory = DummyFactory {
+		client: client.clone(),
+		config: data.link.config.clone(),
+		epoch_changes: data.link.epoch_changes.clone(),
+		mutator: Arc::new(|_, _| ()),
+	};
+
+	// This is just boilerplate code for proposing and importing n valid BABE
+	// blocks that are built on top of the given parent. The proposer takes care
+	// of producing epoch change digests according to the epoch duration (which
+	// is set to 6 slots in the test runtime).
+	let mut propose_and_import_blocks = |parent_id, n| {
+		let mut hashes = Vec::new();
+		let mut parent_header = client.header(&parent_id).unwrap().unwrap();
+
+		for _ in 0..n {
+			let block_hash = propose_and_import_block(
+				&parent_header,
+				None,
+				&mut proposer_factory,
+				&mut block_import,
+			);
+			hashes.push(block_hash);
+			parent_header = client.header(&BlockId::Hash(block_hash)).unwrap().unwrap();
+		}
+
+		hashes
+	};
+
+	// This is the block tree that we're going to use in this test. Each node
+	// represents an epoch change block, the epoch duration is 6 slots.
+	//
+	//    *---- F (#7)
+	//   /                 *------ G (#19) - H (#25)
+	//  /                 /
+	// A (#1) - B (#7) - C (#13) - D (#19) - E (#25)
+	//                              \
+	//                               *------ I (#25)
+
+	// Create and import the canon chain and keep track of fork blocks (A, C, D)
+	// from the diagram above.
+	let canon_hashes = propose_and_import_blocks(BlockId::Number(0), 30);
+
+	// Create the forks
+	let fork_1 = propose_and_import_blocks(BlockId::Hash(canon_hashes[0]), 10);
+	let fork_2 = propose_and_import_blocks(BlockId::Hash(canon_hashes[12]), 15);
+	let fork_3 = propose_and_import_blocks(BlockId::Hash(canon_hashes[18]), 10);
+
+	// We should be tracking a total of 9 epochs in the fork tree
+	assert_eq!(
+		epoch_changes.lock().tree().iter().count(),
+		9,
+	);
+
+	// And only one root
+	assert_eq!(
+		epoch_changes.lock().tree().roots().count(),
+		1,
+	);
+
+	// We finalize block #13 from the canon chain, so on the next epoch
+	// change the tree should be pruned, to not contain F (#7).
+	client.finalize_block(BlockId::Hash(canon_hashes[12]), None, false).unwrap();
+	propose_and_import_blocks(BlockId::Hash(client.info().chain.best_hash), 7);
+
+	// at this point no hashes from the first fork must exist on the tree
+	assert!(
+		!epoch_changes.lock().tree().iter().map(|(h, _, _)| h).any(|h| fork_1.contains(h)),
+	);
+
+	// but the epoch changes from the other forks must still exist
+	assert!(
+		epoch_changes.lock().tree().iter().map(|(h, _, _)| h).any(|h| fork_2.contains(h))
+	);
+
+	assert!(
+		epoch_changes.lock().tree().iter().map(|(h, _, _)| h).any(|h| fork_3.contains(h)),
+	);
+
+	// finalizing block #25 from the canon chain should prune out the second fork
+	client.finalize_block(BlockId::Hash(canon_hashes[24]), None, false).unwrap();
+	propose_and_import_blocks(BlockId::Hash(client.info().chain.best_hash), 8);
+
+	// at this point no hashes from the second fork must exist on the tree
+	assert!(
+		!epoch_changes.lock().tree().iter().map(|(h, _, _)| h).any(|h| fork_2.contains(h)),
+	);
+
+	// while epoch changes from the last fork should still exist
+	assert!(
+		epoch_changes.lock().tree().iter().map(|(h, _, _)| h).any(|h| fork_3.contains(h)),
+	);
+}
+
+#[test]
+#[should_panic]
+fn verify_slots_are_strictly_increasing() {
+	let mut net = BabeTestNet::new(1);
+
+	let peer = net.peer(0);
+	let data = peer.data.as_ref().expect("babe link set up during initialization");
+
+	let client = peer.client().as_full().expect("Only full clients are used in tests").clone();
+	let mut block_import = data.block_import.lock().take().expect("import set up during init");
+
+	let mut proposer_factory = DummyFactory {
+		client: client.clone(),
+		config: data.link.config.clone(),
+		epoch_changes: data.link.epoch_changes.clone(),
+		mutator: Arc::new(|_, _| ()),
+	};
+
+	let genesis_header = client.header(&BlockId::Number(0)).unwrap().unwrap();
+
+	// we should have no issue importing this block
+	let b1 = propose_and_import_block(
+		&genesis_header,
+		Some(999),
+		&mut proposer_factory,
+		&mut block_import,
+	);
+
+	let b1 = client.header(&BlockId::Hash(b1)).unwrap().unwrap();
+
+	// we should fail to import this block since the slot number didn't increase.
+	// we will panic due to the `PanickingBlockImport` defined above.
+	propose_and_import_block(
+		&b1,
+		Some(999),
+		&mut proposer_factory,
+		&mut block_import,
+	);
+}

substrate/client/consensus/babe/src/verification.rs

@@ -0,0 +1,217 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Verification for BABE headers.
+use schnorrkel::vrf::{VRFOutput, VRFProof};
+use sr_primitives::{traits::Header, traits::DigestItemFor};
+use primitives::{Pair, Public};
+use babe_primitives::{Epoch, BabePreDigest, CompatibleDigestItem, AuthorityId};
+use babe_primitives::{AuthoritySignature, SlotNumber, AuthorityIndex, AuthorityPair};
+use slots::CheckedHeader;
+use log::{debug, trace};
+use super::{find_pre_digest, babe_err, BlockT, Error};
+use super::authorship::{make_transcript, calculate_primary_threshold, check_primary_threshold, secondary_slot_author};
+
+/// BABE verification parameters
+pub(super) struct VerificationParams<'a, B: 'a + BlockT> {
+	/// the header being verified.
+	pub(super) header: B::Header,
+	/// the pre-digest of the header being verified. this is optional - if prior
+	/// verification code had to read it, it can be included here to avoid duplicate
+	/// work.
+	pub(super) pre_digest: Option<BabePreDigest>,
+	/// the slot number of the current time.
+	pub(super) slot_now: SlotNumber,
+	/// epoch descriptor of the epoch this block _should_ be under, if it's valid.
+	pub(super) epoch: &'a Epoch,
+	/// genesis config of this BABE chain.
+	pub(super) config: &'a super::Config,
+}
+
+/// Check a header has been signed by the right key. If the slot is too far in
+/// the future, an error will be returned. If successful, returns the pre-header
+/// and the digest item containing the seal.
+///
+/// The seal must be the last digest.  Otherwise, the whole header is considered
+/// unsigned.  This is required for security and must not be changed.
+///
+/// This digest item will always return `Some` when used with `as_babe_pre_digest`.
+///
+/// The given header can either be from a primary or secondary slot assignment,
+/// with each having different validation logic.
+pub(super) fn check_header<B: BlockT + Sized>(
+	params: VerificationParams<B>,
+) -> Result<CheckedHeader<B::Header, VerifiedHeaderInfo<B>>, Error<B>> where
+	DigestItemFor<B>: CompatibleDigestItem,
+{
+	let VerificationParams {
+		mut header,
+		pre_digest,
+		slot_now,
+		epoch,
+		config,
+	} = params;
+
+	let authorities = &epoch.authorities;
+	let pre_digest = pre_digest.map(Ok).unwrap_or_else(|| find_pre_digest::<B>(&header))?;
+
+	trace!(target: "babe", "Checking header");
+	let seal = match header.digest_mut().pop() {
+		Some(x) => x,
+		None => return Err(babe_err(Error::HeaderUnsealed(header.hash()))),
+	};
+
+	let sig = seal.as_babe_seal().ok_or_else(|| {
+		babe_err(Error::HeaderBadSeal(header.hash()))
+	})?;
+
+	// the pre-hash of the header doesn't include the seal
+	// and that's what we sign
+	let pre_hash = header.hash();
+
+	if pre_digest.slot_number() > slot_now {
+		header.digest_mut().push(seal);
+		return Ok(CheckedHeader::Deferred(header, pre_digest.slot_number()));
+	}
+
+	let author = match authorities.get(pre_digest.authority_index() as usize) {
+		Some(author) => author.0.clone(),
+		None => return Err(babe_err(Error::SlotAuthorNotFound)),
+	};
+
+	match &pre_digest {
+		BabePreDigest::Primary { vrf_output, vrf_proof, authority_index, slot_number } => {
+			debug!(target: "babe", "Verifying Primary block");
+
+			let digest = (vrf_output, vrf_proof, *authority_index, *slot_number);
+
+			check_primary_header::<B>(
+				pre_hash,
+				digest,
+				sig,
+				&epoch,
+				config.c,
+			)?;
+		},
+		BabePreDigest::Secondary { authority_index, slot_number } if config.secondary_slots => {
+			debug!(target: "babe", "Verifying Secondary block");
+
+			let digest = (*authority_index, *slot_number);
+
+			check_secondary_header::<B>(
+				pre_hash,
+				digest,
+				sig,
+				&epoch,
+			)?;
+		},
+		_ => {
+			return Err(babe_err(Error::SecondarySlotAssignmentsDisabled));
+		}
+	}
+
+	let info = VerifiedHeaderInfo {
+		pre_digest: CompatibleDigestItem::babe_pre_digest(pre_digest),
+		seal,
+		author,
+	};
+	Ok(CheckedHeader::Checked(header, info))
+}
+
+pub(super) struct VerifiedHeaderInfo<B: BlockT> {
+	pub(super) pre_digest: DigestItemFor<B>,
+	pub(super) seal: DigestItemFor<B>,
+	pub(super) author: AuthorityId,
+}
+
+/// Check a primary slot proposal header. We validate that the given header is
+/// properly signed by the expected authority, and that the contained VRF proof
+/// is valid. Additionally, the weight of this block must increase compared to
+/// its parent since it is a primary block.
+fn check_primary_header<B: BlockT + Sized>(
+	pre_hash: B::Hash,
+	pre_digest: (&VRFOutput, &VRFProof, AuthorityIndex, SlotNumber),
+	signature: AuthoritySignature,
+	epoch: &Epoch,
+	c: (u64, u64),
+) -> Result<(), Error<B>> {
+	let (vrf_output, vrf_proof, authority_index, slot_number) = pre_digest;
+
+	let author = &epoch.authorities[authority_index as usize].0;
+
+	if AuthorityPair::verify(&signature, pre_hash, &author) {
+		let (inout, _) = {
+			let transcript = make_transcript(
+				&epoch.randomness,
+				slot_number,
+				epoch.epoch_index,
+			);
+
+			schnorrkel::PublicKey::from_bytes(author.as_slice()).and_then(|p| {
+				p.vrf_verify(transcript, vrf_output, vrf_proof)
+			}).map_err(|s| {
+				babe_err(Error::VRFVerificationFailed(s))
+			})?
+		};
+
+		let threshold = calculate_primary_threshold(
+			c,
+			&epoch.authorities,
+			authority_index as usize,
+		);
+
+		if !check_primary_threshold(&inout, threshold) {
+			return Err(babe_err(Error::VRFVerificationOfBlockFailed(author.clone(), threshold)));
+		}
+
+		Ok(())
+	} else {
+		Err(babe_err(Error::BadSignature(pre_hash)))
+	}
+}
+
+/// Check a secondary slot proposal header. We validate that the given header is
+/// properly signed by the expected authority, which we have a deterministic way
+/// of computing. Additionally, the weight of this block must stay the same
+/// compared to its parent since it is a secondary block.
+fn check_secondary_header<B: BlockT>(
+	pre_hash: B::Hash,
+	pre_digest: (AuthorityIndex, SlotNumber),
+	signature: AuthoritySignature,
+	epoch: &Epoch,
+) -> Result<(), Error<B>> {
+	let (authority_index, slot_number) = pre_digest;
+
+	// check the signature is valid under the expected authority and
+	// chain state.
+	let expected_author = secondary_slot_author(
+		slot_number,
+		&epoch.authorities,
+		epoch.randomness,
+	).ok_or_else(|| Error::NoSecondaryAuthorExpected)?;
+
+	let author = &epoch.authorities[authority_index as usize].0;
+
+	if expected_author != author {
+		return Err(Error::InvalidAuthor(expected_author.clone(), author.clone()));
+	}
+
+	if AuthorityPair::verify(&signature, pre_hash.as_ref(), author) {
+		Ok(())
+	} else {
+		Err(Error::BadSignature(pre_hash))
+	}
+}

substrate/client/consensus/pow/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "substrate-consensus-pow"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "PoW consensus algorithm for substrate"
+edition = "2018"
+
+[dependencies]
+codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] }
+primitives = { package = "substrate-primitives", path = "../../../primitives/core" }
+sr-primitives = { path = "../../../primitives/sr-primitives" }
+client-api = { package = "substrate-client-api", path = "../../api" }
+block-builder-api = { package = "substrate-block-builder-runtime-api", path = "../../../primitives/block-builder/runtime-api" }
+paint-timestamp = { path = "../../../paint/timestamp" }
+inherents = { package = "substrate-inherents", path = "../../../primitives/inherents" }
+pow-primitives = { package = "substrate-consensus-pow-primitives", path = "../../../primitives/consensus/pow" }
+consensus-common = { package = "substrate-consensus-common", path = "../../../primitives/consensus/common" }
+log = "0.4.8"
+futures-preview = { version = "0.3.0-alpha.19", features = ["compat"] }
+derive_more = "0.15.0"

substrate/client/consensus/pow/src/lib.rs

@@ -0,0 +1,542 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Proof of work consensus for Substrate.
+//!
+//! To use this engine, you can need to have a struct that implements
+//! `PowAlgorithm`. After that, pass an instance of the struct, along
+//! with other necessary client references to `import_queue` to setup
+//! the queue. Use the `start_mine` function for basic CPU mining.
+//!
+//! The auxiliary storage for PoW engine only stores the total difficulty.
+//! For other storage requirements for particular PoW algorithm (such as
+//! the actual difficulty for each particular blocks), you can take a client
+//! reference in your `PowAlgorithm` implementation, and use a separate prefix
+//! for the auxiliary storage. It is also possible to just use the runtime
+//! as the storage, but it is not recommended as it won't work well with light
+//! clients.
+
+use std::sync::Arc;
+use std::thread;
+use std::collections::HashMap;
+use client_api::{
+	BlockOf, blockchain::{HeaderBackend, ProvideCache}, backend::AuxStore,
+	well_known_cache_keys::Id as CacheKeyId,
+};
+use block_builder_api::BlockBuilder as BlockBuilderApi;
+use sr_primitives::{Justification, RuntimeString};
+use sr_primitives::generic::{BlockId, Digest, DigestItem};
+use sr_primitives::traits::{Block as BlockT, Header as HeaderT, ProvideRuntimeApi};
+use paint_timestamp::{TimestampInherentData, InherentError as TIError};
+use pow_primitives::{Seal, TotalDifficulty, POW_ENGINE_ID};
+use primitives::H256;
+use inherents::{InherentDataProviders, InherentData};
+use consensus_common::{
+	BlockImportParams, BlockOrigin, ForkChoiceStrategy, SyncOracle, Environment, Proposer,
+	SelectChain, Error as ConsensusError
+};
+use consensus_common::import_queue::{BoxBlockImport, BasicQueue, Verifier};
+use codec::{Encode, Decode};
+use client_api;
+use log::*;
+
+#[derive(derive_more::Display, Debug)]
+pub enum Error<B: BlockT> {
+	#[display(fmt = "Header uses the wrong engine {:?}", _0)]
+	WrongEngine([u8; 4]),
+	#[display(fmt = "Header {:?} is unsealed", _0)]
+	HeaderUnsealed(B::Hash),
+	#[display(fmt = "PoW validation error: invalid seal")]
+	InvalidSeal,
+	#[display(fmt = "Rejecting block too far in future")]
+	TooFarInFuture,
+	#[display(fmt = "Fetching best header failed using select chain: {:?}", _0)]
+	BestHeaderSelectChain(ConsensusError),
+	#[display(fmt = "Fetching best header failed: {:?}", _0)]
+	BestHeader(client_api::error::Error),
+	#[display(fmt = "Best header does not exist")]
+	NoBestHeader,
+	#[display(fmt = "Block proposing error: {:?}", _0)]
+	BlockProposingError(String),
+	#[display(fmt = "Fetch best hash failed via select chain: {:?}", _0)]
+	BestHashSelectChain(ConsensusError),
+	#[display(fmt = "Error with block built on {:?}: {:?}", _0, _1)]
+	BlockBuiltError(B::Hash, ConsensusError),
+	#[display(fmt = "Creating inherents failed: {}", _0)]
+	CreateInherents(inherents::Error),
+	#[display(fmt = "Checking inherents failed: {}", _0)]
+	CheckInherents(String),
+	Client(client_api::error::Error),
+	Codec(codec::Error),
+	Environment(String),
+	Runtime(RuntimeString)
+}
+
+impl<B: BlockT> std::convert::From<Error<B>> for String {
+	fn from(error: Error<B>) -> String {
+		error.to_string()
+	}
+}
+
+/// Auxiliary storage prefix for PoW engine.
+pub const POW_AUX_PREFIX: [u8; 4] = *b"PoW:";
+
+/// Get the auxiliary storage key used by engine to store total difficulty.
+fn aux_key(hash: &H256) -> Vec<u8> {
+	POW_AUX_PREFIX.iter().chain(&hash[..])
+		.cloned().collect::<Vec<_>>()
+}
+
+/// Auxiliary storage data for PoW.
+#[derive(Encode, Decode, Clone, Debug, Default)]
+pub struct PowAux<Difficulty> {
+	/// Difficulty of the current block.
+	pub difficulty: Difficulty,
+	/// Total difficulty up to current block.
+	pub total_difficulty: Difficulty,
+}
+
+impl<Difficulty> PowAux<Difficulty> where
+	Difficulty: Decode + Default,
+{
+	/// Read the auxiliary from client.
+	pub fn read<C: AuxStore, B: BlockT>(client: &C, hash: &H256) -> Result<Self, Error<B>> {
+		let key = aux_key(hash);
+
+		match client.get_aux(&key).map_err(Error::Client)? {
+			Some(bytes) => Self::decode(&mut &bytes[..])
+				.map_err(Error::Codec),
+			None => Ok(Self::default()),
+		}
+	}
+}
+
+/// Algorithm used for proof of work.
+pub trait PowAlgorithm<B: BlockT> {
+	/// Difficulty for the algorithm.
+	type Difficulty: TotalDifficulty + Default + Encode + Decode + Ord + Clone + Copy;
+
+	/// Get the next block's difficulty.
+	fn difficulty(&self, parent: &BlockId<B>) -> Result<Self::Difficulty, Error<B>>;
+	/// Verify proof of work against the given difficulty.
+	fn verify(
+		&self,
+		parent: &BlockId<B>,
+		pre_hash: &H256,
+		seal: &Seal,
+		difficulty: Self::Difficulty,
+	) -> Result<bool, Error<B>>;
+	/// Mine a seal that satisfies the given difficulty.
+	fn mine(
+		&self,
+		parent: &BlockId<B>,
+		pre_hash: &H256,
+		difficulty: Self::Difficulty,
+		round: u32,
+	) -> Result<Option<Seal>, Error<B>>;
+}
+
+/// A verifier for PoW blocks.
+pub struct PowVerifier<B: BlockT<Hash=H256>, C, S, Algorithm> {
+	client: Arc<C>,
+	algorithm: Algorithm,
+	inherent_data_providers: inherents::InherentDataProviders,
+	select_chain: Option<S>,
+	check_inherents_after: <<B as BlockT>::Header as HeaderT>::Number,
+}
+
+impl<B: BlockT<Hash=H256>, C, S, Algorithm> PowVerifier<B, C, S, Algorithm> {
+	pub fn new(
+		client: Arc<C>,
+		algorithm: Algorithm,
+		check_inherents_after: <<B as BlockT>::Header as HeaderT>::Number,
+		select_chain: Option<S>,
+		inherent_data_providers: inherents::InherentDataProviders,
+	) -> Self {
+		Self { client, algorithm, inherent_data_providers, select_chain, check_inherents_after }
+	}
+
+	fn check_header(
+		&self,
+		mut header: B::Header,
+		parent_block_id: BlockId<B>,
+	) -> Result<(B::Header, Algorithm::Difficulty, DigestItem<H256>), Error<B>> where
+		Algorithm: PowAlgorithm<B>,
+	{
+		let hash = header.hash();
+
+		let (seal, inner_seal) = match header.digest_mut().pop() {
+			Some(DigestItem::Seal(id, seal)) => {
+				if id == POW_ENGINE_ID {
+					(DigestItem::Seal(id, seal.clone()), seal)
+				} else {
+					return Err(Error::WrongEngine(id))
+				}
+			},
+			_ => return Err(Error::HeaderUnsealed(hash)),
+		};
+
+		let pre_hash = header.hash();
+		let difficulty = self.algorithm.difficulty(&parent_block_id)?;
+
+		if !self.algorithm.verify(
+			&parent_block_id,
+			&pre_hash,
+			&inner_seal,
+			difficulty,
+		)? {
+			return Err(Error::InvalidSeal);
+		}
+
+		Ok((header, difficulty, seal))
+	}
+
+	fn check_inherents(
+		&self,
+		block: B,
+		block_id: BlockId<B>,
+		inherent_data: InherentData,
+		timestamp_now: u64,
+	) -> Result<(), Error<B>> where
+		C: ProvideRuntimeApi, C::Api: BlockBuilderApi<B, Error = client_api::error::Error>
+	{
+		const MAX_TIMESTAMP_DRIFT_SECS: u64 = 60;
+
+		if *block.header().number() < self.check_inherents_after {
+			return Ok(())
+		}
+
+		let inherent_res = self.client.runtime_api().check_inherents(
+			&block_id,
+			block,
+			inherent_data,
+		).map_err(Error::Client)?;
+
+		if !inherent_res.ok() {
+			inherent_res
+				.into_errors()
+				.try_for_each(|(i, e)| match TIError::try_from(&i, &e) {
+					Some(TIError::ValidAtTimestamp(timestamp)) => {
+						if timestamp > timestamp_now + MAX_TIMESTAMP_DRIFT_SECS {
+							return Err(Error::TooFarInFuture);
+						}
+
+						Ok(())
+					},
+					Some(TIError::Other(e)) => Err(Error::Runtime(e)),
+					None => Err(Error::CheckInherents(
+						self.inherent_data_providers.error_to_string(&i, &e)
+					)),
+				})
+		} else {
+			Ok(())
+		}
+	}
+}
+
+impl<B: BlockT<Hash=H256>, C, S, Algorithm> Verifier<B> for PowVerifier<B, C, S, Algorithm> where
+	C: ProvideRuntimeApi + Send + Sync + HeaderBackend<B> + AuxStore + ProvideCache<B> + BlockOf,
+	C::Api: BlockBuilderApi<B, Error = client_api::error::Error>,
+	S: SelectChain<B>,
+	Algorithm: PowAlgorithm<B> + Send + Sync,
+{
+	fn verify(
+		&mut self,
+		origin: BlockOrigin,
+		header: B::Header,
+		justification: Option<Justification>,
+		mut body: Option<Vec<B::Extrinsic>>,
+	) -> Result<(BlockImportParams<B>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
+		let inherent_data = self.inherent_data_providers
+			.create_inherent_data().map_err(|e| e.into_string())?;
+		let timestamp_now = inherent_data.timestamp_inherent_data().map_err(|e| e.into_string())?;
+
+		let best_hash = match self.select_chain.as_ref() {
+			Some(select_chain) => select_chain.best_chain()
+				.map_err(|e| format!("Fetch best chain failed via select chain: {:?}", e))?
+				.hash(),
+			None => self.client.info().best_hash,
+		};
+		let hash = header.hash();
+		let parent_hash = *header.parent_hash();
+		let best_aux = PowAux::read::<_, B>(self.client.as_ref(), &best_hash)?;
+		let mut aux = PowAux::read::<_, B>(self.client.as_ref(), &parent_hash)?;
+
+		let (checked_header, difficulty, seal) = self.check_header(
+			header,
+			BlockId::Hash(parent_hash),
+		)?;
+		aux.difficulty = difficulty;
+		aux.total_difficulty.increment(difficulty);
+
+		if let Some(inner_body) = body.take() {
+			let block = B::new(checked_header.clone(), inner_body);
+
+			self.check_inherents(
+				block.clone(),
+				BlockId::Hash(parent_hash),
+				inherent_data,
+				timestamp_now
+			)?;
+
+			let (_, inner_body) = block.deconstruct();
+			body = Some(inner_body);
+		}
+		let key = aux_key(&hash);
+		let import_block = BlockImportParams {
+			origin,
+			header: checked_header,
+			post_digests: vec![seal],
+			body,
+			finalized: false,
+			justification,
+			auxiliary: vec![(key, Some(aux.encode()))],
+			fork_choice: ForkChoiceStrategy::Custom(aux.total_difficulty > best_aux.total_difficulty),
+			allow_missing_state: false,
+		};
+
+		Ok((import_block, None))
+	}
+}
+
+/// Register the PoW inherent data provider, if not registered already.
+pub fn register_pow_inherent_data_provider(
+	inherent_data_providers: &InherentDataProviders,
+) -> Result<(), consensus_common::Error> {
+	if !inherent_data_providers.has_provider(&paint_timestamp::INHERENT_IDENTIFIER) {
+		inherent_data_providers
+			.register_provider(paint_timestamp::InherentDataProvider)
+			.map_err(Into::into)
+			.map_err(consensus_common::Error::InherentData)
+	} else {
+		Ok(())
+	}
+}
+
+/// The PoW import queue type.
+pub type PowImportQueue<B> = BasicQueue<B>;
+
+/// Import queue for PoW engine.
+pub fn import_queue<B, C, S, Algorithm>(
+	block_import: BoxBlockImport<B>,
+	client: Arc<C>,
+	algorithm: Algorithm,
+	check_inherents_after: <<B as BlockT>::Header as HeaderT>::Number,
+	select_chain: Option<S>,
+	inherent_data_providers: InherentDataProviders,
+) -> Result<PowImportQueue<B>, consensus_common::Error> where
+	B: BlockT<Hash=H256>,
+	C: ProvideRuntimeApi + HeaderBackend<B> + BlockOf + ProvideCache<B> + AuxStore,
+	C: Send + Sync + AuxStore + 'static,
+	C::Api: BlockBuilderApi<B, Error = client_api::error::Error>,
+	Algorithm: PowAlgorithm<B> + Send + Sync + 'static,
+	S: SelectChain<B> + 'static,
+{
+	register_pow_inherent_data_provider(&inherent_data_providers)?;
+
+	let verifier = PowVerifier::new(
+		client.clone(),
+		algorithm,
+		check_inherents_after,
+		select_chain,
+		inherent_data_providers,
+	);
+
+	Ok(BasicQueue::new(
+		verifier,
+		block_import,
+		None,
+		None
+	))
+}
+
+/// Start the background mining thread for PoW. Note that because PoW mining
+/// is CPU-intensive, it is not possible to use an async future to define this.
+/// However, it's not recommended to use background threads in the rest of the
+/// codebase.
+///
+/// `preruntime` is a parameter that allows a custom additional pre-runtime
+/// digest to be inserted for blocks being built. This can encode authorship
+/// information, or just be a graffiti. `round` is for number of rounds the
+/// CPU miner runs each time. This parameter should be tweaked so that each
+/// mining round is within sub-second time.
+pub fn start_mine<B: BlockT<Hash=H256>, C, Algorithm, E, SO, S>(
+	mut block_import: BoxBlockImport<B>,
+	client: Arc<C>,
+	algorithm: Algorithm,
+	mut env: E,
+	preruntime: Option<Vec<u8>>,
+	round: u32,
+	mut sync_oracle: SO,
+	build_time: std::time::Duration,
+	select_chain: Option<S>,
+	inherent_data_providers: inherents::InherentDataProviders,
+) where
+	C: HeaderBackend<B> + AuxStore + 'static,
+	Algorithm: PowAlgorithm<B> + Send + Sync + 'static,
+	E: Environment<B> + Send + Sync + 'static,
+	E::Error: std::fmt::Debug,
+	SO: SyncOracle + Send + Sync + 'static,
+	S: SelectChain<B> + 'static,
+{
+	if let Err(_) = register_pow_inherent_data_provider(&inherent_data_providers) {
+		warn!("Registering inherent data provider for timestamp failed");
+	}
+
+	thread::spawn(move || {
+		loop {
+			match mine_loop(
+				&mut block_import,
+				client.as_ref(),
+				&algorithm,
+				&mut env,
+				preruntime.as_ref(),
+				round,
+				&mut sync_oracle,
+				build_time.clone(),
+				select_chain.as_ref(),
+				&inherent_data_providers
+			) {
+				Ok(()) => (),
+				Err(e) => error!(
+					"Mining block failed with {:?}. Sleep for 1 second before restarting...",
+					e
+				),
+			}
+			std::thread::sleep(std::time::Duration::new(1, 0));
+		}
+	});
+}
+
+fn mine_loop<B: BlockT<Hash=H256>, C, Algorithm, E, SO, S>(
+	block_import: &mut BoxBlockImport<B>,
+	client: &C,
+	algorithm: &Algorithm,
+	env: &mut E,
+	preruntime: Option<&Vec<u8>>,
+	round: u32,
+	sync_oracle: &mut SO,
+	build_time: std::time::Duration,
+	select_chain: Option<&S>,
+	inherent_data_providers: &inherents::InherentDataProviders,
+) -> Result<(), Error<B>> where
+	C: HeaderBackend<B> + AuxStore,
+	Algorithm: PowAlgorithm<B>,
+	E: Environment<B>,
+	E::Error: std::fmt::Debug,
+	SO: SyncOracle,
+	S: SelectChain<B>,
+{
+	'outer: loop {
+		if sync_oracle.is_major_syncing() {
+			debug!(target: "pow", "Skipping proposal due to sync.");
+			std::thread::sleep(std::time::Duration::new(1, 0));
+			continue 'outer
+		}
+
+		let (best_hash, best_header) = match select_chain {
+			Some(select_chain) => {
+				let header = select_chain.best_chain()
+					.map_err(Error::BestHeaderSelectChain)?;
+				let hash = header.hash();
+				(hash, header)
+			},
+			None => {
+				let hash = client.info().best_hash;
+				let header = client.header(BlockId::Hash(hash))
+					.map_err(Error::BestHeader)?
+					.ok_or(Error::NoBestHeader)?;
+				(hash, header)
+			},
+		};
+		let mut aux = PowAux::read(client, &best_hash)?;
+		let mut proposer = env.init(&best_header)
+			.map_err(|e| Error::Environment(format!("{:?}", e)))?;
+
+		let inherent_data = inherent_data_providers
+			.create_inherent_data().map_err(Error::CreateInherents)?;
+		let mut inherent_digest = Digest::default();
+		if let Some(preruntime) = &preruntime {
+			inherent_digest.push(DigestItem::PreRuntime(POW_ENGINE_ID, preruntime.to_vec()));
+		}
+		let block = futures::executor::block_on(proposer.propose(
+			inherent_data,
+			inherent_digest,
+			build_time.clone(),
+		)).map_err(|e| Error::BlockProposingError(format!("{:?}", e)))?;
+
+		let (header, body) = block.deconstruct();
+		let (difficulty, seal) = {
+			let difficulty = algorithm.difficulty(
+				&BlockId::Hash(best_hash),
+			)?;
+
+			loop {
+				let seal = algorithm.mine(
+					&BlockId::Hash(best_hash),
+					&header.hash(),
+					difficulty,
+					round,
+				)?;
+
+				if let Some(seal) = seal {
+					break (difficulty, seal)
+				}
+
+				if best_hash != client.info().best_hash {
+					continue 'outer
+				}
+			}
+		};
+
+		aux.difficulty = difficulty;
+		aux.total_difficulty.increment(difficulty);
+		let hash = {
+			let mut header = header.clone();
+			header.digest_mut().push(DigestItem::Seal(POW_ENGINE_ID, seal.clone()));
+			header.hash()
+		};
+
+		let key = aux_key(&hash);
+		let best_hash = match select_chain {
+			Some(select_chain) => select_chain.best_chain()
+				.map_err(Error::BestHashSelectChain)?
+				.hash(),
+			None => client.info().best_hash,
+		};
+		let best_aux = PowAux::<Algorithm::Difficulty>::read(client, &best_hash)?;
+
+		// if the best block has changed in the meantime drop our proposal
+		if best_aux.total_difficulty > aux.total_difficulty {
+			continue 'outer
+		}
+
+		let import_block = BlockImportParams {
+			origin: BlockOrigin::Own,
+			header,
+			justification: None,
+			post_digests: vec![DigestItem::Seal(POW_ENGINE_ID, seal)],
+			body: Some(body),
+			finalized: false,
+			auxiliary: vec![(key, Some(aux.encode()))],
+			fork_choice: ForkChoiceStrategy::Custom(true),
+			allow_missing_state: false,
+		};
+
+		block_import.import_block(import_block, HashMap::default())
+			.map_err(|e| Error::BlockBuiltError(best_hash, e))?;
+	}
+}

substrate/client/consensus/slots/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "substrate-consensus-slots"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Generic slots-based utilities for consensus"
+edition = "2018"
+build = "build.rs"
+
+[dependencies]
+codec = { package = "parity-scale-codec", version = "1.0.0" }
+client-api = { package = "substrate-client-api", path = "../../api" }
+primitives = { package = "substrate-primitives", path = "../../../primitives/core" }
+sr-primitives = {  path = "../../../primitives/sr-primitives" }
+substrate-telemetry = { path = "../../telemetry" }
+consensus_common = { package = "substrate-consensus-common", path = "../../../primitives/consensus/common" }
+inherents = { package = "substrate-inherents", path = "../../../primitives/inherents" }
+futures-preview = "0.3.0-alpha.19"
+futures-timer = "0.4.0"
+parking_lot = "0.9.0"
+log = "0.4.8"
+
+[dev-dependencies]
+test-client = { package = "substrate-test-runtime-client", path = "../../../test/utils/runtime/client" }

substrate/client/consensus/slots/build.rs

@@ -0,0 +1,23 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::env;
+
+fn main() {
+	if let Ok(profile) = env::var("PROFILE") {
+		println!("cargo:rustc-cfg=build_type=\"{}\"", profile);
+	}
+}

substrate/client/consensus/slots/src/aux_schema.rs

@@ -0,0 +1,265 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Schema for slots in the aux-db.
+
+use codec::{Encode, Decode};
+use client_api::backend::AuxStore;
+use client_api::error::{Result as ClientResult, Error as ClientError};
+use sr_primitives::traits::Header;
+
+const SLOT_HEADER_MAP_KEY: &[u8] = b"slot_header_map";
+const SLOT_HEADER_START: &[u8] = b"slot_header_start";
+
+/// We keep at least this number of slots in database.
+pub const MAX_SLOT_CAPACITY: u64 = 1000;
+/// We prune slots when they reach this number.
+pub const PRUNING_BOUND: u64 = 2 * MAX_SLOT_CAPACITY;
+
+fn load_decode<C, T>(backend: &C, key: &[u8]) -> ClientResult<Option<T>>
+	where
+		C: AuxStore,
+		T: Decode,
+{
+	match backend.get_aux(key)? {
+		None => Ok(None),
+		Some(t) => T::decode(&mut &t[..])
+			.map_err(
+				|e| ClientError::Backend(format!("Slots DB is corrupted. Decode error: {}", e.what())),
+			)
+			.map(Some)
+	}
+}
+
+/// Represents an equivocation proof.
+#[derive(Debug, Clone)]
+pub struct EquivocationProof<H> {
+	slot: u64,
+	fst_header: H,
+	snd_header: H,
+}
+
+impl<H> EquivocationProof<H> {
+	/// Get the slot number where the equivocation happened.
+	pub fn slot(&self) -> u64 {
+		self.slot
+	}
+
+	/// Get the first header involved in the equivocation.
+	pub fn fst_header(&self) -> &H {
+		&self.fst_header
+	}
+
+	/// Get the second header involved in the equivocation.
+	pub fn snd_header(&self) -> &H {
+		&self.snd_header
+	}
+}
+
+/// Checks if the header is an equivocation and returns the proof in that case.
+///
+/// Note: it detects equivocations only when slot_now - slot <= MAX_SLOT_CAPACITY.
+pub fn check_equivocation<C, H, P>(
+	backend: &C,
+	slot_now: u64,
+	slot: u64,
+	header: &H,
+	signer: &P,
+) -> ClientResult<Option<EquivocationProof<H>>>
+	where
+		H: Header,
+		C: AuxStore,
+		P: Clone + Encode + Decode + PartialEq,
+{
+	// We don't check equivocations for old headers out of our capacity.
+	if slot_now - slot > MAX_SLOT_CAPACITY {
+		return Ok(None)
+	}
+
+	// Key for this slot.
+	let mut curr_slot_key = SLOT_HEADER_MAP_KEY.to_vec();
+	slot.using_encoded(|s| curr_slot_key.extend(s));
+
+	// Get headers of this slot.
+	let mut headers_with_sig = load_decode::<_, Vec<(H, P)>>(backend, &curr_slot_key[..])?
+		.unwrap_or_else(Vec::new);
+
+	// Get first slot saved.
+	let slot_header_start = SLOT_HEADER_START.to_vec();
+	let first_saved_slot = load_decode::<_, u64>(backend, &slot_header_start[..])?
+		.unwrap_or(slot);
+
+	for (prev_header, prev_signer) in headers_with_sig.iter() {
+		// A proof of equivocation consists of two headers:
+		// 1) signed by the same voter,
+		if prev_signer == signer {
+			// 2) with different hash
+			if header.hash() != prev_header.hash() {
+				return Ok(Some(EquivocationProof {
+					slot, // 3) and mentioning the same slot.
+					fst_header: prev_header.clone(),
+					snd_header: header.clone(),
+				}));
+			} else {
+				//  We don't need to continue in case of duplicated header,
+				// since it's already saved and a possible equivocation
+				// would have been detected before.
+				return Ok(None)
+			}
+		}
+	}
+
+	let mut keys_to_delete = vec![];
+	let mut new_first_saved_slot = first_saved_slot;
+
+	if slot_now - first_saved_slot >= PRUNING_BOUND {
+		let prefix = SLOT_HEADER_MAP_KEY.to_vec();
+		new_first_saved_slot = slot_now.saturating_sub(MAX_SLOT_CAPACITY);
+
+		for s in first_saved_slot..new_first_saved_slot {
+			let mut p = prefix.clone();
+			s.using_encoded(|s| p.extend(s));
+			keys_to_delete.push(p);
+		}
+	}
+
+	headers_with_sig.push((header.clone(), signer.clone()));
+
+	backend.insert_aux(
+		&[
+			(&curr_slot_key[..], headers_with_sig.encode().as_slice()),
+			(&slot_header_start[..], new_first_saved_slot.encode().as_slice()),
+		],
+		&keys_to_delete.iter().map(|k| &k[..]).collect::<Vec<&[u8]>>()[..],
+	)?;
+
+	Ok(None)
+}
+
+#[cfg(test)]
+mod test {
+	use primitives::{sr25519, Pair};
+	use primitives::hash::H256;
+	use sr_primitives::testing::{Header as HeaderTest, Digest as DigestTest};
+	use test_client;
+
+	use super::{MAX_SLOT_CAPACITY, PRUNING_BOUND, check_equivocation};
+
+	fn create_header(number: u64) -> HeaderTest {
+		// so that different headers for the same number get different hashes
+		let parent_hash = H256::random();
+
+		let header = HeaderTest {
+			parent_hash,
+			number,
+			state_root: Default::default(),
+			extrinsics_root: Default::default(),
+			digest: DigestTest { logs: vec![], },
+		};
+
+		header
+	}
+
+	#[test]
+	fn check_equivocation_works() {
+		let client = test_client::new();
+		let (pair, _seed) = sr25519::Pair::generate();
+		let public = pair.public();
+
+		let header1 = create_header(1); // @ slot 2
+		let header2 = create_header(2); // @ slot 2
+		let header3 = create_header(2); // @ slot 4
+		let header4 = create_header(3); // @ slot MAX_SLOT_CAPACITY + 4
+		let header5 = create_header(4); // @ slot MAX_SLOT_CAPACITY + 4
+		let header6 = create_header(3); // @ slot 4
+
+		// It's ok to sign same headers.
+		assert!(
+			check_equivocation(
+				&client,
+				2,
+				2,
+				&header1,
+				&public,
+			).unwrap().is_none(),
+		);
+
+		assert!(
+			check_equivocation(
+				&client,
+				3,
+				2,
+				&header1,
+				&public,
+			).unwrap().is_none(),
+		);
+
+		// But not two different headers at the same slot.
+		assert!(
+			check_equivocation(
+				&client,
+				4,
+				2,
+				&header2,
+				&public,
+			).unwrap().is_some(),
+		);
+
+		// Different slot is ok.
+		assert!(
+			check_equivocation(
+				&client,
+				5,
+				4,
+				&header3,
+				&public,
+			).unwrap().is_none(),
+		);
+
+		// Here we trigger pruning and save header 4.
+		assert!(
+			check_equivocation(
+				&client,
+				PRUNING_BOUND + 2,
+				MAX_SLOT_CAPACITY + 4,
+				&header4,
+				&public,
+			).unwrap().is_none(),
+		);
+
+		// This fails because header 5 is an equivocation of header 4.
+		assert!(
+			check_equivocation(
+				&client,
+				PRUNING_BOUND + 3,
+				MAX_SLOT_CAPACITY + 4,
+				&header5,
+				&public,
+			).unwrap().is_some(),
+		);
+
+		// This is ok because we pruned the corresponding header. Shows that we are pruning.
+		assert!(
+			check_equivocation(
+				&client,
+				PRUNING_BOUND + 4,
+				4,
+				&header6,
+				&public,
+			).unwrap().is_none(),
+		);
+	}
+}

substrate/client/consensus/slots/src/lib.rs

@@ -0,0 +1,430 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Slots functionality for Substrate.
+//!
+//! Some consensus algorithms have a concept of *slots*, which are intervals in
+//! time during which certain events can and/or must occur.  This crate
+//! provides generic functionality for slots.
+
+#![deny(warnings)]
+#![forbid(unsafe_code, missing_docs)]
+
+mod slots;
+mod aux_schema;
+
+pub use slots::{SignedDuration, SlotInfo};
+use slots::Slots;
+pub use aux_schema::{check_equivocation, MAX_SLOT_CAPACITY, PRUNING_BOUND};
+
+use codec::{Decode, Encode};
+use consensus_common::{BlockImport, Proposer, SyncOracle, SelectChain};
+use futures::{prelude::*, future::{self, Either}};
+use futures_timer::Delay;
+use inherents::{InherentData, InherentDataProviders};
+use log::{debug, error, info, warn};
+use sr_primitives::generic::BlockId;
+use sr_primitives::traits::{ApiRef, Block as BlockT, Header, ProvideRuntimeApi};
+use std::{fmt::Debug, ops::Deref, pin::Pin, sync::Arc};
+use substrate_telemetry::{telemetry, CONSENSUS_DEBUG, CONSENSUS_WARN, CONSENSUS_INFO};
+use parking_lot::Mutex;
+use client_api;
+
+/// A worker that should be invoked at every new slot.
+pub trait SlotWorker<B: BlockT> {
+	/// The type of the future that will be returned when a new slot is
+	/// triggered.
+	type OnSlot: Future<Output = Result<(), consensus_common::Error>>;
+
+	/// Called when a new slot is triggered.
+	fn on_slot(&mut self, chain_head: B::Header, slot_info: SlotInfo) -> Self::OnSlot;
+}
+
+/// A skeleton implementation for `SlotWorker` which tries to claim a slot at
+/// its beginning and tries to produce a block if successfully claimed, timing
+/// out if block production takes too long.
+pub trait SimpleSlotWorker<B: BlockT> {
+	/// A handle to a `BlockImport`.
+	type BlockImport: BlockImport<B> + Send + 'static;
+
+	/// A handle to a `SyncOracle`.
+	type SyncOracle: SyncOracle;
+
+	/// The type of proposer to use to build blocks.
+	type Proposer: Proposer<B>;
+
+	/// Data associated with a slot claim.
+	type Claim: Send + 'static;
+
+	/// Epoch data necessary for authoring.
+	type EpochData;
+
+	/// The logging target to use when logging messages.
+	fn logging_target(&self) -> &'static str;
+
+	/// A handle to a `BlockImport`.
+	fn block_import(&self) -> Arc<Mutex<Self::BlockImport>>;
+
+	/// Returns the epoch data necessary for authoring. For time-dependent epochs,
+	/// use the provided slot number as a canonical source of time.
+	fn epoch_data(&self, header: &B::Header, slot_number: u64) -> Result<Self::EpochData, consensus_common::Error>;
+
+	/// Returns the number of authorities given the epoch data.
+	fn authorities_len(&self, epoch_data: &Self::EpochData) -> usize;
+
+	/// Tries to claim the given slot, returning an object with claim data if successful.
+	fn claim_slot(
+		&self,
+		header: &B::Header,
+		slot_number: u64,
+		epoch_data: &Self::EpochData,
+	) -> Option<Self::Claim>;
+
+	/// Return the pre digest data to include in a block authored with the given claim.
+	fn pre_digest_data(&self, slot_number: u64, claim: &Self::Claim) -> Vec<sr_primitives::DigestItem<B::Hash>>;
+
+	/// Returns a function which produces a `BlockImportParams`.
+	fn block_import_params(&self) -> Box<dyn Fn(
+		B::Header,
+		&B::Hash,
+		Vec<B::Extrinsic>,
+		Self::Claim,
+	) -> consensus_common::BlockImportParams<B> + Send>;
+
+	/// Whether to force authoring if offline.
+	fn force_authoring(&self) -> bool;
+
+	/// Returns a handle to a `SyncOracle`.
+	fn sync_oracle(&mut self) -> &mut Self::SyncOracle;
+
+	/// Returns a `Proposer` to author on top of the given block.
+	fn proposer(&mut self, block: &B::Header) -> Result<Self::Proposer, consensus_common::Error>;
+
+	/// Implements the `on_slot` functionality from `SlotWorker`.
+	fn on_slot(&mut self, chain_head: B::Header, slot_info: SlotInfo)
+		-> Pin<Box<dyn Future<Output = Result<(), consensus_common::Error>> + Send>> where
+		Self: Send + Sync,
+		<Self::Proposer as Proposer<B>>::Create: Unpin + Send + 'static,
+	{
+		let (timestamp, slot_number, slot_duration) =
+			(slot_info.timestamp, slot_info.number, slot_info.duration);
+
+		let epoch_data = match self.epoch_data(&chain_head, slot_number) {
+			Ok(epoch_data) => epoch_data,
+			Err(err) => {
+				warn!("Unable to fetch epoch data at block {:?}: {:?}", chain_head.hash(), err);
+
+				telemetry!(
+					CONSENSUS_WARN; "slots.unable_fetching_authorities";
+					"slot" => ?chain_head.hash(),
+					"err" => ?err,
+				);
+
+				return Box::pin(future::ready(Ok(())));
+			}
+		};
+
+		let authorities_len = self.authorities_len(&epoch_data);
+
+		if !self.force_authoring() && self.sync_oracle().is_offline() && authorities_len > 1 {
+			debug!(target: self.logging_target(), "Skipping proposal slot. Waiting for the network.");
+			telemetry!(
+				CONSENSUS_DEBUG;
+				"slots.skipping_proposal_slot";
+				"authorities_len" => authorities_len,
+			);
+
+			return Box::pin(future::ready(Ok(())));
+		}
+
+		let claim = match self.claim_slot(&chain_head, slot_number, &epoch_data) {
+			None => return Box::pin(future::ready(Ok(()))),
+			Some(claim) => claim,
+		};
+
+		debug!(
+			target: self.logging_target(), "Starting authorship at slot {}; timestamp = {}",
+			slot_number,
+			timestamp,
+		);
+
+		telemetry!(CONSENSUS_DEBUG; "slots.starting_authorship";
+			"slot_num" => slot_number,
+			"timestamp" => timestamp,
+		);
+
+		let mut proposer = match self.proposer(&chain_head) {
+			Ok(proposer) => proposer,
+			Err(err) => {
+				warn!("Unable to author block in slot {:?}: {:?}", slot_number, err);
+
+				telemetry!(CONSENSUS_WARN; "slots.unable_authoring_block";
+					"slot" => slot_number, "err" => ?err
+				);
+
+				return Box::pin(future::ready(Ok(())));
+			},
+		};
+
+		let remaining_duration = slot_info.remaining_duration();
+		let logs = self.pre_digest_data(slot_number, &claim);
+
+		// deadline our production to approx. the end of the slot
+		let proposal_work = futures::future::select(
+			proposer.propose(
+				slot_info.inherent_data,
+				sr_primitives::generic::Digest {
+					logs,
+				},
+				remaining_duration,
+			).map_err(|e| consensus_common::Error::ClientImport(format!("{:?}", e))),
+			Delay::new(remaining_duration)
+				.map_err(consensus_common::Error::FaultyTimer)
+		).map(|v| match v {
+			futures::future::Either::Left((b, _)) => b.map(|b| (b, claim)),
+			futures::future::Either::Right((Ok(_), _)) =>
+				Err(consensus_common::Error::ClientImport("Timeout in the Slots proposer".into())),
+			futures::future::Either::Right((Err(err), _)) => Err(err),
+		});
+
+		let block_import_params_maker = self.block_import_params();
+		let block_import = self.block_import();
+		let logging_target = self.logging_target();
+
+		Box::pin(proposal_work.map_ok(move |(block, claim)| {
+			// minor hack since we don't have access to the timestamp
+			// that is actually set by the proposer.
+			let slot_after_building = SignedDuration::default().slot_now(slot_duration);
+			if slot_after_building != slot_number {
+				info!("Discarding proposal for slot {}; block production took too long", slot_number);
+				// If the node was compiled with debug, tell the user to use release optimizations.
+				#[cfg(build_type="debug")]
+				info!("Recompile your node in `--release` mode to mitigate this problem.");
+				telemetry!(CONSENSUS_INFO; "slots.discarding_proposal_took_too_long";
+					"slot" => slot_number,
+				);
+
+				return;
+			}
+
+			let (header, body) = block.deconstruct();
+			let header_num = *header.number();
+			let header_hash = header.hash();
+			let parent_hash = *header.parent_hash();
+
+			let block_import_params = block_import_params_maker(
+				header,
+				&header_hash,
+				body,
+				claim,
+			);
+
+			info!("Pre-sealed block for proposal at {}. Hash now {:?}, previously {:?}.",
+					header_num,
+					block_import_params.post_header().hash(),
+					header_hash,
+			);
+
+			telemetry!(CONSENSUS_INFO; "slots.pre_sealed_block";
+				"header_num" => ?header_num,
+				"hash_now" => ?block_import_params.post_header().hash(),
+				"hash_previously" => ?header_hash,
+			);
+
+			if let Err(err) = block_import.lock().import_block(block_import_params, Default::default()) {
+				warn!(target: logging_target,
+					"Error with block built on {:?}: {:?}",
+					parent_hash,
+					err,
+				);
+
+				telemetry!(CONSENSUS_WARN; "slots.err_with_block_built_on";
+					"hash" => ?parent_hash, "err" => ?err,
+				);
+			}
+		}))
+	}
+}
+
+/// Slot compatible inherent data.
+pub trait SlotCompatible {
+	/// Extract timestamp and slot from inherent data.
+	fn extract_timestamp_and_slot(
+		&self,
+		inherent: &InherentData,
+	) -> Result<(u64, u64, std::time::Duration), consensus_common::Error>;
+
+	/// Get the difference between chain time and local time.  Defaults to
+	/// always returning zero.
+	fn time_offset() -> SignedDuration { Default::default() }
+}
+
+/// Start a new slot worker.
+///
+/// Every time a new slot is triggered, `worker.on_slot` is called and the future it returns is
+/// polled until completion, unless we are major syncing.
+pub fn start_slot_worker<B, C, W, T, SO, SC>(
+	slot_duration: SlotDuration<T>,
+	client: C,
+	mut worker: W,
+	mut sync_oracle: SO,
+	inherent_data_providers: InherentDataProviders,
+	timestamp_extractor: SC,
+) -> impl Future<Output = ()>
+where
+	B: BlockT,
+	C: SelectChain<B> + Clone,
+	W: SlotWorker<B>,
+	W::OnSlot: Unpin,
+	SO: SyncOracle + Send + Clone,
+	SC: SlotCompatible + Unpin,
+	T: SlotData + Clone,
+{
+	let SlotDuration(slot_duration) = slot_duration;
+
+	// rather than use a timer interval, we schedule our waits ourselves
+	Slots::<SC>::new(
+		slot_duration.slot_duration(),
+		inherent_data_providers,
+		timestamp_extractor,
+	).inspect_err(|e| debug!(target: "slots", "Faulty timer: {:?}", e))
+		.try_for_each(move |slot_info| {
+			// only propose when we are not syncing.
+			if sync_oracle.is_major_syncing() {
+				debug!(target: "slots", "Skipping proposal slot due to sync.");
+				return Either::Right(future::ready(Ok(())));
+			}
+
+			let slot_num = slot_info.number;
+			let chain_head = match client.best_chain() {
+				Ok(x) => x,
+				Err(e) => {
+					warn!(target: "slots", "Unable to author block in slot {}. \
+					no best block header: {:?}", slot_num, e);
+					return Either::Right(future::ready(Ok(())));
+				}
+			};
+
+			Either::Left(worker.on_slot(chain_head, slot_info).map_err(
+				|e| {
+					warn!(target: "slots", "Encountered consensus error: {:?}", e);
+				}).or_else(|_| future::ready(Ok(())))
+			)
+		}).then(|res| {
+			if let Err(err) = res {
+				warn!(target: "slots", "Slots stream terminated with an error: {:?}", err);
+			}
+			future::ready(())
+		})
+}
+
+/// A header which has been checked
+pub enum CheckedHeader<H, S> {
+	/// A header which has slot in the future. this is the full header (not stripped)
+	/// and the slot in which it should be processed.
+	Deferred(H, u64),
+	/// A header which is fully checked, including signature. This is the pre-header
+	/// accompanied by the seal components.
+	///
+	/// Includes the digest item that encoded the seal.
+	Checked(H, S),
+}
+
+/// A type from which a slot duration can be obtained.
+pub trait SlotData {
+	/// Gets the slot duration.
+	fn slot_duration(&self) -> u64;
+
+	/// The static slot key
+	const SLOT_KEY: &'static [u8];
+}
+
+impl SlotData for u64 {
+	fn slot_duration(&self) -> u64 {
+		*self
+	}
+
+	const SLOT_KEY: &'static [u8] = b"aura_slot_duration";
+}
+
+/// A slot duration. Create with `get_or_compute`.
+// The internal member should stay private here to maintain invariants of
+// `get_or_compute`.
+#[derive(Clone, Copy, Debug, Encode, Decode, Hash, PartialOrd, Ord, PartialEq, Eq)]
+pub struct SlotDuration<T>(T);
+
+impl<T> Deref for SlotDuration<T> {
+	type Target = T;
+	fn deref(&self) -> &T {
+		&self.0
+	}
+}
+
+impl<T: SlotData + Clone> SlotData for SlotDuration<T> {
+	/// Get the slot duration in milliseconds.
+	fn slot_duration(&self) -> u64
+		where T: SlotData,
+	{
+		self.0.slot_duration()
+	}
+
+	const SLOT_KEY: &'static [u8] = T::SLOT_KEY;
+}
+
+impl<T: Clone> SlotDuration<T> {
+	/// Either fetch the slot duration from disk or compute it from the
+	/// genesis state.
+	///
+	/// `slot_key` is marked as `'static`, as it should really be a
+	/// compile-time constant.
+	pub fn get_or_compute<B: BlockT, C, CB>(client: &C, cb: CB) -> client_api::error::Result<Self> where
+		C: client_api::backend::AuxStore,
+		C: ProvideRuntimeApi,
+		CB: FnOnce(ApiRef<C::Api>, &BlockId<B>) -> client_api::error::Result<T>,
+		T: SlotData + Encode + Decode + Debug,
+	{
+		match client.get_aux(T::SLOT_KEY)? {
+			Some(v) => <T as codec::Decode>::decode(&mut &v[..])
+				.map(SlotDuration)
+				.map_err(|_| {
+					client_api::error::Error::Backend({
+						error!(target: "slots", "slot duration kept in invalid format");
+						"slot duration kept in invalid format".to_string()
+					})
+				}),
+			None => {
+				use sr_primitives::traits::Zero;
+				let genesis_slot_duration =
+					cb(client.runtime_api(), &BlockId::number(Zero::zero()))?;
+
+				info!(
+					"Loaded block-time = {:?} milliseconds from genesis on first-launch",
+					genesis_slot_duration
+				);
+
+				genesis_slot_duration
+					.using_encoded(|s| client.insert_aux(&[(T::SLOT_KEY, &s[..])], &[]))?;
+
+				Ok(SlotDuration(genesis_slot_duration))
+			}
+		}
+	}
+
+	/// Returns slot data value.
+	pub fn get(&self) -> T {
+		self.0.clone()
+	}
+}

substrate/client/consensus/slots/src/slots.rs

@@ -0,0 +1,176 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Utility stream for yielding slots in a loop.
+//!
+//! This is used instead of `futures_timer::Interval` because it was unreliable.
+
+use super::SlotCompatible;
+use consensus_common::Error;
+use futures::{prelude::*, task::Context, task::Poll};
+use inherents::{InherentData, InherentDataProviders};
+
+use std::{pin::Pin, time::{Duration, Instant}};
+use futures_timer::Delay;
+
+/// Returns current duration since unix epoch.
+pub fn duration_now() -> Duration {
+	use std::time::SystemTime;
+	let now = SystemTime::now();
+	now.duration_since(SystemTime::UNIX_EPOCH).unwrap_or_else(|e| panic!(
+		"Current time {:?} is before unix epoch. Something is wrong: {:?}",
+		now,
+		e,
+	))
+}
+
+
+/// A `Duration` with a sign (before or after).  Immutable.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
+pub struct SignedDuration {
+	offset: Duration,
+	is_positive: bool,
+}
+
+impl SignedDuration {
+	/// Construct a `SignedDuration`
+	pub fn new(offset: Duration, is_positive: bool) -> Self {
+		Self { offset, is_positive }
+	}
+
+	/// Get the slot for now.  Panics if `slot_duration` is 0.
+	pub fn slot_now(&self, slot_duration: u64) -> u64 {
+		(if self.is_positive {
+			duration_now() + self.offset
+		} else {
+			duration_now() - self.offset
+		}.as_millis() as u64) / slot_duration
+	}
+}
+
+/// Returns the duration until the next slot, based on current duration since
+pub fn time_until_next(now: Duration, slot_duration: u64) -> Duration {
+	let remaining_full_millis = slot_duration - (now.as_millis() as u64 % slot_duration) - 1;
+	Duration::from_millis(remaining_full_millis)
+}
+
+/// Information about a slot.
+pub struct SlotInfo {
+	/// The slot number.
+	pub number: u64,
+	/// Current timestamp.
+	pub timestamp: u64,
+	/// The instant at which the slot ends.
+	pub ends_at: Instant,
+	/// The inherent data.
+	pub inherent_data: InherentData,
+	/// Slot duration.
+	pub duration: u64,
+}
+
+impl SlotInfo {
+	/// Yields the remaining duration in the slot.
+	pub fn remaining_duration(&self) -> Duration {
+		let now = Instant::now();
+		if now < self.ends_at {
+			self.ends_at.duration_since(now)
+		} else {
+			Duration::from_millis(0)
+		}
+	}
+}
+
+/// A stream that returns every time there is a new slot.
+pub(crate) struct Slots<SC> {
+	last_slot: u64,
+	slot_duration: u64,
+	inner_delay: Option<Delay>,
+	inherent_data_providers: InherentDataProviders,
+	timestamp_extractor: SC,
+}
+
+impl<SC> Slots<SC> {
+	/// Create a new `Slots` stream.
+	pub fn new(
+		slot_duration: u64,
+		inherent_data_providers: InherentDataProviders,
+		timestamp_extractor: SC,
+	) -> Self {
+		Slots {
+			last_slot: 0,
+			slot_duration,
+			inner_delay: None,
+			inherent_data_providers,
+			timestamp_extractor,
+		}
+	}
+}
+
+impl<SC: SlotCompatible + Unpin> Stream for Slots<SC> {
+	type Item = Result<SlotInfo, Error>;
+
+	fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
+		loop {
+			let slot_duration = self.slot_duration;
+			self.inner_delay = match self.inner_delay.take() {
+				None => {
+					// schedule wait.
+					let wait_dur = time_until_next(duration_now(), slot_duration);
+					Some(Delay::new(wait_dur))
+				}
+				Some(d) => Some(d),
+			};
+
+			if let Some(ref mut inner_delay) = self.inner_delay {
+				match Future::poll(Pin::new(inner_delay), cx) {
+					Poll::Pending => return Poll::Pending,
+					Poll::Ready(Err(err)) => return Poll::Ready(Some(Err(Error::FaultyTimer(err)))),
+					Poll::Ready(Ok(())) => {}
+				}
+			}
+
+			// timeout has fired.
+
+			let inherent_data = match self.inherent_data_providers.create_inherent_data() {
+				Ok(id) => id,
+				Err(err) => return Poll::Ready(Some(Err(consensus_common::Error::InherentData(err)))),
+			};
+			let result = self.timestamp_extractor.extract_timestamp_and_slot(&inherent_data);
+			let (timestamp, slot_num, offset) = match result {
+				Ok(v) => v,
+				Err(err) => return Poll::Ready(Some(Err(err))),
+			};
+			// reschedule delay for next slot.
+			let ends_in = offset +
+				time_until_next(Duration::from_millis(timestamp), slot_duration);
+			let ends_at = Instant::now() + ends_in;
+			self.inner_delay = Some(Delay::new(ends_in));
+
+			// never yield the same slot twice.
+			if slot_num > self.last_slot {
+				self.last_slot = slot_num;
+
+				break Poll::Ready(Some(Ok(SlotInfo {
+					number: slot_num,
+					duration: self.slot_duration,
+					timestamp,
+					ends_at,
+					inherent_data,
+				})))
+			}
+		}
+	}
+}

substrate/client/consensus/uncles/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "substrate-consensus-uncles"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Generic uncle inclusion utilities for consensus"
+edition = "2018"
+
+[dependencies]
+client-api = { package = "substrate-client-api", path = "../../api" }
+primitives = { package = "substrate-primitives", path = "../../../primitives/core" }
+sr-primitives = {  path = "../../../primitives/sr-primitives" }
+paint-authorship = {  path = "../../../paint/authorship" }
+consensus_common = { package = "substrate-consensus-common", path = "../../../primitives/consensus/common" }
+inherents = { package = "substrate-inherents", path = "../../../primitives/inherents" }
+log = "0.4.8"

substrate/client/consensus/uncles/src/lib.rs

@@ -0,0 +1,66 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Uncles functionality for Substrate.
+//!
+#![deny(warnings)]
+#![forbid(unsafe_code, missing_docs)]
+
+use consensus_common::SelectChain;
+use inherents::{InherentDataProviders};
+use log::warn;
+use client_api::ProvideUncles;
+use sr_primitives::traits::{Block as BlockT, Header};
+use std::sync::Arc;
+
+/// Maximum uncles generations we may provide to the runtime.
+const MAX_UNCLE_GENERATIONS: u32 = 8;
+
+/// Register uncles inherent data provider, if not registered already.
+pub fn register_uncles_inherent_data_provider<B, C, SC>(
+	client: Arc<C>,
+	select_chain: SC,
+	inherent_data_providers: &InherentDataProviders,
+) -> Result<(), consensus_common::Error> where
+	B: BlockT,
+	C: ProvideUncles<B> + Send + Sync + 'static,
+	SC: SelectChain<B> + 'static,
+{
+	if !inherent_data_providers.has_provider(&paint_authorship::INHERENT_IDENTIFIER) {
+		inherent_data_providers
+			.register_provider(paint_authorship::InherentDataProvider::new(move || {
+				{
+					let chain_head = match select_chain.best_chain() {
+						Ok(x) => x,
+						Err(e) => {
+							warn!(target: "uncles", "Unable to get chain head: {:?}", e);
+							return Vec::new();
+						}
+					};
+					match client.uncles(chain_head.hash(), MAX_UNCLE_GENERATIONS.into()) {
+						Ok(uncles) => uncles,
+						Err(e) => {
+							warn!(target: "uncles", "Unable to get uncles: {:?}", e);
+							Vec::new()
+						}
+					}
+				}
+			}))
+		.map_err(|err| consensus_common::Error::InherentData(err.into()))?;
+	}
+	Ok(())
+}
+

substrate/client/db/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "substrate-client-db"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+parking_lot = "0.9.0"
+log = "0.4.8"
+kvdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
+# FIXME replace with release as soon as our rocksdb changes are released upstream https://github.com/paritytech/parity-common/issues/88
+kvdb-rocksdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d", optional = true }
+kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
+linked-hash-map = "0.5.2"
+hash-db = "0.15.2"
+client-api = { package = "substrate-client-api", path = "../api" }
+primitives = { package = "substrate-primitives", path = "../../primitives/core" }
+sr-primitives = {  path = "../../primitives/sr-primitives" }
+client = { package = "substrate-client", path = "../" }
+state-machine = { package = "substrate-state-machine", path = "../../primitives/state-machine" }
+codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] }
+executor = { package = "substrate-executor", path = "../executor" }
+state_db = { package = "substrate-state-db", path = "../state-db" }
+trie = { package = "substrate-trie", path = "../../primitives/trie" }
+consensus_common = { package = "substrate-consensus-common", path = "../../primitives/consensus/common" }
+header_metadata = { package = "substrate-header-metadata", path = "../header-metadata" }
+
+[dev-dependencies]
+substrate-keyring = { path = "../../primitives/keyring" }
+test-client = { package = "substrate-test-runtime-client", path = "../../test/utils/runtime/client" }
+env_logger = "0.7.0"
+quickcheck = "0.8"
+
+[features]
+default = []
+test-helpers = []

substrate/client/db/src/cache/list_entry.rs

@@ -0,0 +1,161 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! List-cache storage entries.
+
+use client_api::error::Result as ClientResult;
+use sr_primitives::traits::{Block as BlockT, NumberFor};
+use codec::{Encode, Decode};
+
+use crate::cache::{CacheItemT, ComplexBlockId};
+use crate::cache::list_storage::{Storage};
+
+/// Single list-based cache entry.
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+pub struct Entry<Block: BlockT, T> {
+	/// first block, when this value became actual.
+	pub valid_from: ComplexBlockId<Block>,
+	/// Value stored at this entry.
+	pub value: T,
+}
+
+/// Internal representation of the single list-based cache entry. The entry points to the
+/// previous entry in the cache, allowing us to traverse back in time in list-style.
+#[derive(Debug, Encode, Decode)]
+#[cfg_attr(test, derive(Clone, PartialEq))]
+pub struct StorageEntry<Block: BlockT, T: CacheItemT> {
+	/// None if valid from the beginning.
+	pub prev_valid_from: Option<ComplexBlockId<Block>>,
+	/// Value stored at this entry.
+	pub value: T,
+}
+
+impl<Block: BlockT, T: CacheItemT> Entry<Block, T> {
+	/// Returns Some if the entry should be updated with the new value.
+	pub fn try_update(&self, value: Option<T>) -> Option<StorageEntry<Block, T>> {
+		match value {
+			Some(value) => match self.value == value {
+				true => None,
+				false => Some(StorageEntry {
+					prev_valid_from: Some(self.valid_from.clone()),
+					value,
+				}),
+			},
+			None => None,
+		}
+	}
+
+	/// Wrapper that calls search_before to get range where the given block fits.
+	pub fn search_best_range_before<S: Storage<Block, T>>(
+		&self,
+		storage: &S,
+		block: NumberFor<Block>,
+	) -> ClientResult<Option<(ComplexBlockId<Block>, Option<ComplexBlockId<Block>>)>> {
+		Ok(self.search_best_before(storage, block)?
+			.map(|(entry, next)| (entry.valid_from, next)))
+	}
+
+	/// Searches the list, ending with THIS entry for the best entry preceeding (or at)
+	/// given block number.
+	/// If the entry is found, result is the entry and the block id of next entry (if exists).
+	/// NOTE that this function does not check that the passed block is actually linked to
+	/// the blocks it found.
+	pub fn search_best_before<S: Storage<Block, T>>(
+		&self,
+		storage: &S,
+		block: NumberFor<Block>,
+	) -> ClientResult<Option<(Entry<Block, T>, Option<ComplexBlockId<Block>>)>> {
+		// we're looking for the best value
+		let mut next = None;
+		let mut current = self.valid_from.clone();
+		if block >= self.valid_from.number {
+			let value = self.value.clone();
+			return Ok(Some((Entry { valid_from: current, value }, next)));
+		}
+
+		// else - travel back in time
+		loop {
+			let entry = storage.require_entry(&current)?;
+			if block >= current.number {
+				return Ok(Some((Entry { valid_from: current, value: entry.value }, next)));
+			}
+
+			next = Some(current);
+			current = match entry.prev_valid_from {
+				Some(prev_valid_from) => prev_valid_from,
+				None => return Ok(None),
+			};
+		}
+	}
+}
+
+impl<Block: BlockT, T: CacheItemT> StorageEntry<Block, T> {
+	/// Converts storage entry into an entry, valid from given block.
+	pub fn into_entry(self, valid_from: ComplexBlockId<Block>) -> Entry<Block, T> {
+		Entry {
+			valid_from,
+			value: self.value,
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use crate::cache::list_cache::tests::test_id;
+	use crate::cache::list_storage::tests::{DummyStorage, FaultyStorage};
+	use super::*;
+
+	#[test]
+	fn entry_try_update_works() {
+		// when trying to update with None value
+		assert_eq!(Entry::<_, u64> { valid_from: test_id(1), value: 42 }.try_update(None), None);
+		// when trying to update with the same Some value
+		assert_eq!(Entry { valid_from: test_id(1), value: 1 }.try_update(Some(1)), None);
+		// when trying to update with different Some value
+		assert_eq!(Entry { valid_from: test_id(1), value: 1 }.try_update(Some(2)),
+			Some(StorageEntry { prev_valid_from: Some(test_id(1)), value: 2 }));
+	}
+
+	#[test]
+	fn entry_search_best_before_fails() {
+		// when storage returns error
+		assert!(Entry::<_, u64> { valid_from: test_id(100), value: 42 }
+			.search_best_before(&FaultyStorage, 50).is_err());
+	}
+
+	#[test]
+	fn entry_search_best_before_works() {
+		// when block is better than our best block
+		assert_eq!(Entry::<_, u64> { valid_from: test_id(100), value: 100 }
+			.search_best_before(&DummyStorage::new(), 150).unwrap(),
+		Some((Entry::<_, u64> { valid_from: test_id(100), value: 100 }, None)));
+		// when block is found between two entries
+		assert_eq!(Entry::<_, u64> { valid_from: test_id(100), value: 100 }
+			.search_best_before(&DummyStorage::new()
+				.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 })
+				.with_entry(test_id(50), StorageEntry { prev_valid_from: Some(test_id(30)), value: 50 }),
+			75).unwrap(),
+		Some((Entry::<_, u64> { valid_from: test_id(50), value: 50 }, Some(test_id(100)))));
+		// when block is not found
+		assert_eq!(Entry::<_, u64> { valid_from: test_id(100), value: 100 }
+			.search_best_before(&DummyStorage::new()
+				.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 })
+				.with_entry(test_id(50), StorageEntry { prev_valid_from: None, value: 50 }),
+			30).unwrap(),
+		None);
+	}
+}

substrate/client/db/src/cache/list_storage.rs

@@ -0,0 +1,386 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! List-cache storage definition and implementation.
+
+use std::sync::Arc;
+
+use kvdb::{KeyValueDB, DBTransaction};
+
+use client_api::error::{Error as ClientError, Result as ClientResult};
+use codec::{Encode, Decode};
+use sr_primitives::generic::BlockId;
+use sr_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor};
+use crate::utils::{self, db_err, meta_keys};
+
+use crate::cache::{CacheItemT, ComplexBlockId};
+use crate::cache::list_cache::{CommitOperation, Fork};
+use crate::cache::list_entry::{Entry, StorageEntry};
+
+/// Single list-cache metadata.
+#[derive(Debug)]
+#[cfg_attr(test, derive(Clone, PartialEq))]
+pub struct Metadata<Block: BlockT> {
+	/// Block at which best finalized entry is stored.
+	pub finalized: Option<ComplexBlockId<Block>>,
+	/// A set of blocks at which best unfinalized entries are stored.
+	pub unfinalized: Vec<ComplexBlockId<Block>>,
+}
+
+/// Readonly list-cache storage trait.
+pub trait Storage<Block: BlockT, T: CacheItemT> {
+	/// Reads hash of the block at given number.
+	fn read_id(&self, at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>>;
+
+	/// Reads header of the block with given hash.
+	fn read_header(&self, at: &Block::Hash) -> ClientResult<Option<Block::Header>>;
+
+	/// Reads cache metadata: best finalized entry (if some) and the list.
+	fn read_meta(&self) -> ClientResult<Metadata<Block>>;
+
+	/// Reads cache entry from the storage.
+	fn read_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>>;
+
+	/// Reads referenced (and thus existing) cache entry from the storage.
+	fn require_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<StorageEntry<Block, T>> {
+		self.read_entry(at)
+			.and_then(|entry| entry
+				.ok_or_else(|| ClientError::from(
+					ClientError::Backend(format!("Referenced cache entry at {:?} is not found", at)))))
+	}
+}
+
+/// List-cache storage transaction.
+pub trait StorageTransaction<Block: BlockT, T: CacheItemT> {
+	/// Insert storage entry at given block.
+	fn insert_storage_entry(&mut self, at: &ComplexBlockId<Block>, entry: &StorageEntry<Block, T>);
+
+	/// Delete storage entry at given block.
+	fn remove_storage_entry(&mut self, at: &ComplexBlockId<Block>);
+
+	/// Update metadata of the cache.
+	fn update_meta(
+		&mut self,
+		best_finalized_entry: Option<&Entry<Block, T>>,
+		unfinalized: &[Fork<Block, T>],
+		operation: &CommitOperation<Block, T>,
+	);
+}
+
+/// A set of columns used by the DbStorage.
+#[derive(Debug)]
+pub struct DbColumns {
+	/// Column holding cache meta.
+	pub meta: Option<u32>,
+	/// Column holding the mapping of { block number => block hash } for blocks of the best chain.
+	pub key_lookup: Option<u32>,
+	/// Column holding the mapping of { block hash => block header }.
+	pub header: Option<u32>,
+	/// Column holding cache entries.
+	pub cache: Option<u32>,
+}
+
+/// Database-backed list cache storage.
+pub struct DbStorage {
+	name: Vec<u8>,
+	meta_key: Vec<u8>,
+	db: Arc<dyn KeyValueDB>,
+	columns: DbColumns,
+}
+
+impl DbStorage {
+	/// Create new database-backed list cache storage.
+	pub fn new(name: Vec<u8>, db: Arc<dyn KeyValueDB>, columns: DbColumns) -> Self {
+		let meta_key = meta::key(&name);
+		DbStorage { name, meta_key, db, columns }
+	}
+
+	/// Get reference to the database.
+	pub fn db(&self) -> &Arc<dyn KeyValueDB> { &self.db }
+
+	/// Get reference to the database columns.
+	pub fn columns(&self) -> &DbColumns { &self.columns }
+
+	/// Encode block id for storing as a key in cache column.
+	/// We append prefix to the actual encoding to allow several caches
+	/// store entries in the same column.
+	pub fn encode_block_id<Block: BlockT>(&self, block: &ComplexBlockId<Block>) -> Vec<u8> {
+		let mut encoded = self.name.clone();
+		encoded.extend(block.hash.as_ref());
+		encoded
+	}
+}
+
+impl<Block: BlockT, T: CacheItemT> Storage<Block, T> for DbStorage {
+	fn read_id(&self, at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>> {
+		utils::read_header::<Block>(&*self.db, self.columns.key_lookup, self.columns.header, BlockId::Number(at))
+			.map(|maybe_header| maybe_header.map(|header| header.hash()))
+	}
+
+	fn read_header(&self, at: &Block::Hash) -> ClientResult<Option<Block::Header>> {
+		utils::read_header::<Block>(&*self.db, self.columns.key_lookup, self.columns.header, BlockId::Hash(*at))
+	}
+
+	fn read_meta(&self) -> ClientResult<Metadata<Block>> {
+		self.db.get(self.columns.meta, &self.meta_key)
+			.map_err(db_err)
+			.and_then(|meta| match meta {
+				Some(meta) => meta::decode(&*meta),
+				None => Ok(Metadata {
+					finalized: None,
+					unfinalized: Vec::new(),
+				}),
+			})
+	}
+
+	fn read_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>> {
+		self.db.get(self.columns.cache, &self.encode_block_id(at))
+			.map_err(db_err)
+			.and_then(|entry| match entry {
+				Some(entry) => StorageEntry::<Block, T>::decode(&mut &entry[..])
+					.map_err(|_| ClientError::Backend("Failed to decode cache entry".into()))
+					.map(Some),
+				None => Ok(None),
+			})
+	}
+}
+
+/// Database-backed list cache storage transaction.
+pub struct DbStorageTransaction<'a> {
+	storage: &'a DbStorage,
+	tx: &'a mut DBTransaction,
+}
+
+impl<'a> DbStorageTransaction<'a> {
+	/// Create new database transaction.
+	pub fn new(storage: &'a DbStorage, tx: &'a mut DBTransaction) -> Self {
+		DbStorageTransaction { storage, tx }
+	}
+}
+
+impl<'a, Block: BlockT, T: CacheItemT> StorageTransaction<Block, T> for DbStorageTransaction<'a> {
+	fn insert_storage_entry(&mut self, at: &ComplexBlockId<Block>, entry: &StorageEntry<Block, T>) {
+		self.tx.put(self.storage.columns.cache, &self.storage.encode_block_id(at), &entry.encode());
+	}
+
+	fn remove_storage_entry(&mut self, at: &ComplexBlockId<Block>) {
+		self.tx.delete(self.storage.columns.cache, &self.storage.encode_block_id(at));
+	}
+
+	fn update_meta(
+		&mut self,
+		best_finalized_entry: Option<&Entry<Block, T>>,
+		unfinalized: &[Fork<Block, T>],
+		operation: &CommitOperation<Block, T>,
+	) {
+		self.tx.put(
+			self.storage.columns.meta,
+			&self.storage.meta_key,
+			&meta::encode(best_finalized_entry, unfinalized, operation));
+	}
+}
+
+/// Metadata related functions.
+mod meta {
+	use super::*;
+
+	/// Convert cache name into cache metadata key.
+	pub fn key(name: &[u8]) -> Vec<u8> {
+		let mut key_name = meta_keys::CACHE_META_PREFIX.to_vec();
+		key_name.extend_from_slice(name);
+		key_name
+	}
+
+	/// Encode cache metadata 'applying' commit operation before encoding.
+	pub fn encode<Block: BlockT, T: CacheItemT>(
+		best_finalized_entry: Option<&Entry<Block, T>>,
+		unfinalized: &[Fork<Block, T>],
+		op: &CommitOperation<Block, T>
+	) -> Vec<u8> {
+		let mut finalized = best_finalized_entry.as_ref().map(|entry| &entry.valid_from);
+		let mut unfinalized = unfinalized.iter().map(|fork| &fork.head().valid_from).collect::<Vec<_>>();
+
+		match op {
+			CommitOperation::AppendNewBlock(_, _) => (),
+			CommitOperation::AppendNewEntry(index, ref entry) => {
+				unfinalized[*index] = &entry.valid_from;
+			},
+			CommitOperation::AddNewFork(ref entry) => {
+				unfinalized.push(&entry.valid_from);
+			},
+			CommitOperation::BlockFinalized(_, ref finalizing_entry, ref forks) => {
+				finalized = finalizing_entry.as_ref().map(|entry| &entry.valid_from);
+				for fork_index in forks.iter().rev() {
+					unfinalized.remove(*fork_index);
+				}
+			},
+			CommitOperation::BlockReverted(ref forks) => {
+				for (fork_index, updated_fork) in forks.iter().rev() {
+					match updated_fork {
+						Some(updated_fork) => unfinalized[*fork_index] = &updated_fork.head().valid_from,
+						None => { unfinalized.remove(*fork_index); },
+					}
+				}
+			},
+		}
+
+		(finalized, unfinalized).encode()
+	}
+
+	/// Decode meta information.
+	pub fn decode<Block: BlockT>(encoded: &[u8]) -> ClientResult<Metadata<Block>> {
+		let input = &mut &*encoded;
+		let finalized: Option<ComplexBlockId<Block>> = Decode::decode(input)
+			.map_err(|_| ClientError::from(ClientError::Backend("Error decoding cache meta".into())))?;
+		let unfinalized: Vec<ComplexBlockId<Block>> = Decode::decode(input)
+			.map_err(|_| ClientError::from(ClientError::Backend("Error decoding cache meta".into())))?;
+
+		Ok(Metadata { finalized, unfinalized })
+	}
+}
+
+#[cfg(test)]
+pub mod tests {
+	use std::collections::{HashMap, HashSet};
+	use super::*;
+
+	pub struct FaultyStorage;
+
+	impl<Block: BlockT, T: CacheItemT> Storage<Block, T> for FaultyStorage {
+		fn read_id(&self, _at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>> {
+			Err(ClientError::Backend("TestError".into()))
+		}
+
+		fn read_header(&self, _at: &Block::Hash) -> ClientResult<Option<Block::Header>> {
+			Err(ClientError::Backend("TestError".into()))
+		}
+
+		fn read_meta(&self) -> ClientResult<Metadata<Block>> {
+			Err(ClientError::Backend("TestError".into()))
+		}
+
+		fn read_entry(&self, _at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>> {
+			Err(ClientError::Backend("TestError".into()))
+		}
+	}
+
+	pub struct DummyStorage<Block: BlockT, T: CacheItemT> {
+		meta: Metadata<Block>,
+		ids: HashMap<NumberFor<Block>, Block::Hash>,
+		headers: HashMap<Block::Hash, Block::Header>,
+		entries: HashMap<Block::Hash, StorageEntry<Block, T>>,
+	}
+
+	impl<Block: BlockT, T: CacheItemT> DummyStorage<Block, T> {
+		pub fn new() -> Self {
+			DummyStorage {
+				meta: Metadata {
+					finalized: None,
+					unfinalized: Vec::new(),
+				},
+				ids: HashMap::new(),
+				headers: HashMap::new(),
+				entries: HashMap::new(),
+			}
+		}
+
+		pub fn with_meta(mut self, finalized: Option<ComplexBlockId<Block>>, unfinalized: Vec<ComplexBlockId<Block>>) -> Self {
+			self.meta.finalized = finalized;
+			self.meta.unfinalized = unfinalized;
+			self
+		}
+
+		pub fn with_id(mut self, at: NumberFor<Block>, id: Block::Hash) -> Self {
+			self.ids.insert(at, id);
+			self
+		}
+
+		pub fn with_header(mut self, header: Block::Header) -> Self {
+			self.headers.insert(header.hash(), header);
+			self
+		}
+
+		pub fn with_entry(mut self, at: ComplexBlockId<Block>, entry: StorageEntry<Block, T>) -> Self {
+			self.entries.insert(at.hash, entry);
+			self
+		}
+	}
+
+	impl<Block: BlockT, T: CacheItemT> Storage<Block, T> for DummyStorage<Block, T> {
+		fn read_id(&self, at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>> {
+			Ok(self.ids.get(&at).cloned())
+		}
+
+		fn read_header(&self, at: &Block::Hash) -> ClientResult<Option<Block::Header>> {
+			Ok(self.headers.get(&at).cloned())
+		}
+
+		fn read_meta(&self) -> ClientResult<Metadata<Block>> {
+			Ok(self.meta.clone())
+		}
+
+		fn read_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>> {
+			Ok(self.entries.get(&at.hash).cloned())
+		}
+	}
+
+	pub struct DummyTransaction<Block: BlockT> {
+		updated_meta: Option<Metadata<Block>>,
+		inserted_entries: HashSet<Block::Hash>,
+		removed_entries: HashSet<Block::Hash>,
+	}
+
+	impl<Block: BlockT> DummyTransaction<Block> {
+		pub fn new() -> Self {
+			DummyTransaction {
+				updated_meta: None,
+				inserted_entries: HashSet::new(),
+				removed_entries: HashSet::new(),
+			}
+		}
+
+		pub fn inserted_entries(&self) -> &HashSet<Block::Hash> {
+			&self.inserted_entries
+		}
+
+		pub fn removed_entries(&self) -> &HashSet<Block::Hash> {
+			&self.removed_entries
+		}
+
+		pub fn updated_meta(&self) -> &Option<Metadata<Block>> {
+			&self.updated_meta
+		}
+	}
+
+	impl<Block: BlockT, T: CacheItemT> StorageTransaction<Block, T> for DummyTransaction<Block> {
+		fn insert_storage_entry(&mut self, at: &ComplexBlockId<Block>, _entry: &StorageEntry<Block, T>) {
+			self.inserted_entries.insert(at.hash);
+		}
+
+		fn remove_storage_entry(&mut self, at: &ComplexBlockId<Block>) {
+			self.removed_entries.insert(at.hash);
+		}
+
+		fn update_meta(
+			&mut self,
+			best_finalized_entry: Option<&Entry<Block, T>>,
+			unfinalized: &[Fork<Block, T>],
+			operation: &CommitOperation<Block, T>,
+		) {
+			self.updated_meta = Some(meta::decode(&meta::encode(best_finalized_entry, unfinalized, operation)).unwrap());
+		}
+	}
+}

substrate/client/db/src/cache/mod.rs

@@ -0,0 +1,371 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! DB-backed cache of blockchain data.
+
+use std::{sync::Arc, collections::HashMap};
+use parking_lot::RwLock;
+
+use kvdb::{KeyValueDB, DBTransaction};
+
+use client_api::blockchain::{well_known_cache_keys::{self, Id as CacheKeyId}, Cache as BlockchainCache};
+use client_api::error::Result as ClientResult;
+use codec::{Encode, Decode};
+use sr_primitives::generic::BlockId;
+use sr_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero};
+use crate::utils::{self, COLUMN_META, db_err};
+
+use self::list_cache::{ListCache, PruningStrategy};
+
+mod list_cache;
+mod list_entry;
+mod list_storage;
+
+/// Minimal post-finalization age of finalized blocks before they'll pruned.
+const PRUNE_DEPTH: u32 = 1024;
+
+/// The type of entry that is inserted to the cache.
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum EntryType {
+	/// Non-final entry.
+	NonFinal,
+	/// Final entry.
+	Final,
+	/// Genesis entry (inserted during cache initialization).
+	Genesis,
+}
+
+/// Block identifier that holds both hash and number.
+#[derive(Clone, Debug, Encode, Decode, PartialEq)]
+pub struct ComplexBlockId<Block: BlockT> {
+	hash: Block::Hash,
+	number: NumberFor<Block>,
+}
+
+impl<Block: BlockT> ComplexBlockId<Block> {
+	/// Create new complex block id.
+	pub fn new(hash: Block::Hash, number: NumberFor<Block>) -> Self {
+		ComplexBlockId { hash, number }
+	}
+}
+
+impl<Block: BlockT> ::std::cmp::PartialOrd for ComplexBlockId<Block> {
+	fn partial_cmp(&self, other: &ComplexBlockId<Block>) -> Option<::std::cmp::Ordering> {
+		self.number.partial_cmp(&other.number)
+	}
+}
+
+/// All cache items must implement this trait.
+pub trait CacheItemT: Clone + Decode + Encode + PartialEq {}
+
+impl<T> CacheItemT for T where T: Clone + Decode + Encode + PartialEq {}
+
+/// Database-backed blockchain data cache.
+pub struct DbCache<Block: BlockT> {
+	cache_at: HashMap<CacheKeyId, ListCache<Block, Vec<u8>, self::list_storage::DbStorage>>,
+	db: Arc<dyn KeyValueDB>,
+	key_lookup_column: Option<u32>,
+	header_column: Option<u32>,
+	authorities_column: Option<u32>,
+	genesis_hash: Block::Hash,
+	best_finalized_block: ComplexBlockId<Block>,
+}
+
+impl<Block: BlockT> DbCache<Block> {
+	/// Create new cache.
+	pub fn new(
+		db: Arc<dyn KeyValueDB>,
+		key_lookup_column: Option<u32>,
+		header_column: Option<u32>,
+		authorities_column: Option<u32>,
+		genesis_hash: Block::Hash,
+		best_finalized_block: ComplexBlockId<Block>,
+	) -> Self {
+		Self {
+			cache_at: HashMap::new(),
+			db,
+			key_lookup_column,
+			header_column,
+			authorities_column,
+			genesis_hash,
+			best_finalized_block,
+		}
+	}
+
+	/// Set genesis block hash.
+	pub fn set_genesis_hash(&mut self, genesis_hash: Block::Hash) {
+		self.genesis_hash = genesis_hash;
+	}
+
+	/// Begin cache transaction.
+	pub fn transaction<'a>(&'a mut self, tx: &'a mut DBTransaction) -> DbCacheTransaction<'a, Block> {
+		DbCacheTransaction {
+			cache: self,
+			tx,
+			cache_at_op: HashMap::new(),
+			best_finalized_block: None,
+		}
+	}
+
+	/// Run post-commit cache operations.
+	pub fn commit(&mut self, ops: DbCacheTransactionOps<Block>) {
+		for (name, op) in ops.cache_at_op.into_iter() {
+			self.get_cache(name).on_transaction_commit(op);
+		}
+		if let Some(best_finalized_block) = ops.best_finalized_block {
+			self.best_finalized_block = best_finalized_block;
+		}
+	}
+
+	/// Creates `ListCache` with the given name or returns a reference to the existing.
+	fn get_cache(&mut self, name: CacheKeyId) -> &mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage> {
+		get_cache_helper(
+			&mut self.cache_at,
+			name,
+			&self.db,
+			self.key_lookup_column,
+			self.header_column,
+			self.authorities_column,
+			&self.best_finalized_block
+		)
+	}
+}
+
+// This helper is needed because otherwise the borrow checker will require to
+// clone all parameters outside of the closure.
+fn get_cache_helper<'a, Block: BlockT>(
+	cache_at: &'a mut HashMap<CacheKeyId, ListCache<Block, Vec<u8>, self::list_storage::DbStorage>>,
+	name: CacheKeyId,
+	db: &Arc<dyn KeyValueDB>,
+	key_lookup: Option<u32>,
+	header: Option<u32>,
+	cache: Option<u32>,
+	best_finalized_block: &ComplexBlockId<Block>,
+) -> &'a mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage> {
+	cache_at.entry(name).or_insert_with(|| {
+		ListCache::new(
+			self::list_storage::DbStorage::new(name.to_vec(), db.clone(),
+				self::list_storage::DbColumns {
+					meta: COLUMN_META,
+					key_lookup,
+					header,
+					cache,
+				},
+			),
+			cache_pruning_strategy(name),
+			best_finalized_block.clone(),
+		)
+	})
+}
+
+/// Cache operations that are to be committed after database transaction is committed.
+pub struct DbCacheTransactionOps<Block: BlockT> {
+	cache_at_op: HashMap<CacheKeyId, self::list_cache::CommitOperation<Block, Vec<u8>>>,
+	best_finalized_block: Option<ComplexBlockId<Block>>,
+}
+
+/// Database-backed blockchain data cache transaction valid for single block import.
+pub struct DbCacheTransaction<'a, Block: BlockT> {
+	cache: &'a mut DbCache<Block>,
+	tx: &'a mut DBTransaction,
+	cache_at_op: HashMap<CacheKeyId, self::list_cache::CommitOperation<Block, Vec<u8>>>,
+	best_finalized_block: Option<ComplexBlockId<Block>>,
+}
+
+impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
+	/// Convert transaction into post-commit operations set.
+	pub fn into_ops(self) -> DbCacheTransactionOps<Block> {
+		DbCacheTransactionOps {
+			cache_at_op: self.cache_at_op,
+			best_finalized_block: self.best_finalized_block,
+		}
+	}
+
+	/// When new block is inserted into database.
+	pub fn on_block_insert(
+		mut self,
+		parent: ComplexBlockId<Block>,
+		block: ComplexBlockId<Block>,
+		data_at: HashMap<CacheKeyId, Vec<u8>>,
+		entry_type: EntryType,
+	) -> ClientResult<Self> {
+		assert!(self.cache_at_op.is_empty());
+
+		// prepare list of caches that are not update
+		// (we might still need to do some cache maintenance in this case)
+		let missed_caches = self.cache.cache_at.keys()
+			.filter(|cache| !data_at.contains_key(*cache))
+			.cloned()
+			.collect::<Vec<_>>();
+
+		let mut insert_op = |name: CacheKeyId, value: Option<Vec<u8>>| -> Result<(), client_api::error::Error> {
+			let cache = self.cache.get_cache(name);
+			let op = cache.on_block_insert(
+				&mut self::list_storage::DbStorageTransaction::new(
+					cache.storage(),
+					&mut self.tx,
+				),
+				parent.clone(),
+				block.clone(),
+				value,
+				entry_type,
+			)?;
+			if let Some(op) = op {
+				self.cache_at_op.insert(name, op);
+			}
+			Ok(())
+		};
+
+		data_at.into_iter().try_for_each(|(name, data)| insert_op(name, Some(data)))?;
+		missed_caches.into_iter().try_for_each(|name| insert_op(name, None))?;
+
+		match entry_type {
+			EntryType::Final | EntryType::Genesis =>
+				self.best_finalized_block = Some(block),
+			EntryType::NonFinal => (),
+		}
+
+		Ok(self)
+	}
+
+	/// When previously inserted block is finalized.
+	pub fn on_block_finalize(
+		mut self,
+		parent: ComplexBlockId<Block>,
+		block: ComplexBlockId<Block>
+	) -> ClientResult<Self> {
+		assert!(self.cache_at_op.is_empty());
+
+		for (name, cache_at) in self.cache.cache_at.iter() {
+			let op = cache_at.on_block_finalize(
+				&mut self::list_storage::DbStorageTransaction::new(
+					cache_at.storage(),
+					&mut self.tx
+				),
+				parent.clone(),
+				block.clone(),
+			)?;
+
+			if let Some(op) = op {
+				self.cache_at_op.insert(name.to_owned(), op);
+			}
+		}
+
+		self.best_finalized_block = Some(block);
+
+		Ok(self)
+	}
+
+	/// When block is reverted.
+	pub fn on_block_revert(
+		mut self,
+		reverted_block: &ComplexBlockId<Block>,
+	) -> ClientResult<Self> {
+		for (name, cache) in self.cache.cache_at.iter() {
+			let op = cache.on_block_revert(
+				&mut self::list_storage::DbStorageTransaction::new(
+					cache.storage(),
+					&mut self.tx
+				),
+				reverted_block,
+			)?;
+
+			assert!(!self.cache_at_op.contains_key(name));
+			self.cache_at_op.insert(name.to_owned(), op);
+		}
+
+		Ok(self)
+	}
+}
+
+/// Synchronous implementation of database-backed blockchain data cache.
+pub struct DbCacheSync<Block: BlockT>(pub RwLock<DbCache<Block>>);
+
+impl<Block: BlockT> BlockchainCache<Block> for DbCacheSync<Block> {
+	fn initialize(&self, key: &CacheKeyId, data: Vec<u8>) -> ClientResult<()> {
+		let mut cache = self.0.write();
+		let genesis_hash = cache.genesis_hash;
+		let cache_contents = vec![(*key, data)].into_iter().collect();
+		let db = cache.db.clone();
+		let mut dbtx = DBTransaction::new();
+		let tx = cache.transaction(&mut dbtx);
+		let tx = tx.on_block_insert(
+			ComplexBlockId::new(Default::default(), Zero::zero()),
+			ComplexBlockId::new(genesis_hash, Zero::zero()),
+			cache_contents,
+			EntryType::Genesis,
+		)?;
+		let tx_ops = tx.into_ops();
+		db.write(dbtx).map_err(db_err)?;
+		cache.commit(tx_ops);
+		Ok(())
+	}
+
+	fn get_at(
+		&self,
+		key: &CacheKeyId,
+		at: &BlockId<Block>,
+	) -> Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)> {
+		let mut cache = self.0.write();
+		let storage = cache.get_cache(*key).storage();
+		let db = storage.db();
+		let columns = storage.columns();
+		let at = match *at {
+			BlockId::Hash(hash) => {
+				let header = utils::read_header::<Block>(
+					&**db,
+					columns.key_lookup,
+					columns.header,
+					BlockId::Hash(hash.clone())).ok()??;
+				ComplexBlockId::new(hash, *header.number())
+			},
+			BlockId::Number(number) => {
+				let hash = utils::read_header::<Block>(
+					&**db,
+					columns.key_lookup,
+					columns.header,
+					BlockId::Number(number.clone())).ok()??.hash();
+				ComplexBlockId::new(hash, number)
+			},
+		};
+
+		cache.cache_at
+			.get(key)?
+			.value_at_block(&at)
+			.map(|block_and_value| block_and_value.map(|(begin_block, end_block, value)|
+				(
+					(begin_block.number, begin_block.hash),
+					end_block.map(|end_block| (end_block.number, end_block.hash)),
+					value,
+				)))
+			.ok()?
+	}
+}
+
+/// Get pruning strategy for given cache.
+fn cache_pruning_strategy<N: From<u32>>(cache: CacheKeyId) -> PruningStrategy<N> {
+	// the cache is mostly used to store data from consensus engines
+	// this kind of data is only required for non-finalized blocks
+	// => by default we prune finalized cached entries
+
+	match cache {
+		// we need to keep changes tries configurations forever (or at least until changes tries,
+		// that were built using this configuration, are pruned) to make it possible to refer
+		// to old changes tries
+		well_known_cache_keys::CHANGES_TRIE_CONFIG => PruningStrategy::NeverPrune,
+		_ => PruningStrategy::ByDepth(PRUNE_DEPTH.into()),
+	}
+}

substrate/client/db/src/children.rs

@@ -0,0 +1,121 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Functionality for reading and storing children hashes from db.
+
+use kvdb::{KeyValueDB, DBTransaction};
+use codec::{Encode, Decode};
+use client_api::error;
+use std::hash::Hash;
+
+
+/// Returns the hashes of the children blocks of the block with `parent_hash`.
+pub fn read_children<
+	K: Eq + Hash + Clone + Encode + Decode,
+	V: Eq + Hash + Clone + Encode + Decode,
+>(db: &dyn KeyValueDB, column: Option<u32>, prefix: &[u8], parent_hash: K) -> error::Result<Vec<V>> {
+	let mut buf = prefix.to_vec();
+	parent_hash.using_encoded(|s| buf.extend(s));
+
+	let raw_val_opt = match db.get(column, &buf[..]) {
+		Ok(raw_val_opt) => raw_val_opt,
+		Err(_) => return Err(error::Error::Backend("Error reading value from database".into())),
+	};
+
+	let raw_val = match raw_val_opt {
+		Some(val) => val,
+		None => return Ok(Vec::new()),
+	};
+
+	let children: Vec<V> = match Decode::decode(&mut &raw_val[..]) {
+		Ok(children) => children,
+		Err(_) => return Err(error::Error::Backend("Error decoding children".into())),
+	};
+
+	Ok(children)
+}
+
+/// Insert the key-value pair (`parent_hash`, `children_hashes`) in the transaction.
+/// Any existing value is overwritten upon write.
+pub fn write_children<
+	K: Eq + Hash + Clone + Encode + Decode,
+	V: Eq + Hash + Clone + Encode + Decode,
+>(
+	tx: &mut DBTransaction,
+	column: Option<u32>,
+	prefix: &[u8],
+	parent_hash: K,
+	children_hashes: V,
+) {
+	let mut key = prefix.to_vec();
+	parent_hash.using_encoded(|s| key.extend(s));
+	tx.put_vec(column, &key[..], children_hashes.encode());
+}
+
+/// Prepare transaction to remove the children of `parent_hash`.
+pub fn remove_children<
+	K: Eq + Hash + Clone + Encode + Decode,
+>(
+	tx: &mut DBTransaction,
+	column: Option<u32>,
+	prefix: &[u8],
+	parent_hash: K,
+) {
+	let mut key = prefix.to_vec();
+	parent_hash.using_encoded(|s| key.extend(s));
+	tx.delete(column, &key[..]);
+}
+
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn children_write_read_remove() {
+		const PREFIX: &[u8] = b"children";
+		let db = ::kvdb_memorydb::create(0);
+
+		let mut tx = DBTransaction::new();
+
+		let mut children1 = Vec::new();
+		children1.push(1_3);
+		children1.push(1_5);
+		write_children(&mut tx, None, PREFIX, 1_1, children1);
+
+		let mut children2 = Vec::new();
+		children2.push(1_4);
+		children2.push(1_6);
+		write_children(&mut tx, None, PREFIX, 1_2, children2);
+
+		db.write(tx.clone()).unwrap();
+
+		let r1: Vec<u32> = read_children(&db, None, PREFIX, 1_1).unwrap();
+		let r2: Vec<u32> = read_children(&db, None, PREFIX, 1_2).unwrap();
+
+		assert_eq!(r1, vec![1_3, 1_5]);
+		assert_eq!(r2, vec![1_4, 1_6]);
+
+		remove_children(&mut tx, None, PREFIX, 1_2);
+		db.write(tx).unwrap();
+
+		let r1: Vec<u32> = read_children(&db, None, PREFIX, 1_1).unwrap();
+		let r2: Vec<u32> = read_children(&db, None, PREFIX, 1_2).unwrap();
+
+		assert_eq!(r1, vec![1_3, 1_5]);
+		assert_eq!(r2.len(), 0);
+	}
+}

substrate/client/db/src/offchain.rs

@@ -0,0 +1,148 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! RocksDB-based offchain workers local storage.
+
+use std::{
+	collections::HashMap,
+	sync::Arc,
+};
+
+use crate::columns;
+use kvdb::KeyValueDB;
+use parking_lot::Mutex;
+
+/// Offchain local storage
+#[derive(Clone)]
+pub struct LocalStorage {
+	db: Arc<dyn KeyValueDB>,
+	locks: Arc<Mutex<HashMap<Vec<u8>, Arc<Mutex<()>>>>>,
+}
+
+impl std::fmt::Debug for LocalStorage {
+	fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+		fmt.debug_struct("LocalStorage")
+			.finish()
+	}
+}
+
+impl LocalStorage {
+	/// Create new offchain storage for tests (backed by memorydb)
+	#[cfg(any(test, feature = "test-helpers"))]
+	pub fn new_test() -> Self {
+		let db = Arc::new(::kvdb_memorydb::create(crate::utils::NUM_COLUMNS));
+		Self::new(db as _)
+	}
+
+	/// Create offchain local storage with given `KeyValueDB` backend.
+	pub fn new(db: Arc<dyn KeyValueDB>) -> Self {
+		Self {
+			db,
+			locks: Default::default(),
+		}
+	}
+}
+
+impl client_api::OffchainStorage for LocalStorage {
+	fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) {
+		let key: Vec<u8> = prefix.iter().chain(key).cloned().collect();
+		let mut tx = self.db.transaction();
+		tx.put(columns::OFFCHAIN, &key, value);
+
+		if let Err(e) = self.db.write(tx) {
+			log::warn!("Error writing to the offchain DB: {:?}", e);
+		}
+	}
+
+	fn get(&self, prefix: &[u8], key: &[u8]) -> Option<Vec<u8>> {
+		let key: Vec<u8> = prefix.iter().chain(key).cloned().collect();
+		self.db.get(columns::OFFCHAIN, &key)
+			.ok()
+			.and_then(|x| x)
+			.map(|v| v.to_vec())
+	}
+
+	fn compare_and_set(
+		&mut self,
+		prefix: &[u8],
+		item_key: &[u8],
+		old_value: Option<&[u8]>,
+		new_value: &[u8],
+	) -> bool {
+		let key: Vec<u8> = prefix.iter().chain(item_key).cloned().collect();
+		let key_lock = {
+			let mut locks = self.locks.lock();
+			locks.entry(key.clone()).or_default().clone()
+		};
+
+		let is_set;
+		{
+			let _key_guard = key_lock.lock();
+			let val = self.db.get(columns::OFFCHAIN, &key)
+				.ok()
+				.and_then(|x| x);
+			is_set = val.as_ref().map(|x| &**x) == old_value;
+
+			if is_set {
+				self.set(prefix, item_key, new_value)
+			}
+		}
+
+		// clean the lock map if we're the only entry
+		let mut locks = self.locks.lock();
+		{
+			drop(key_lock);
+			let key_lock = locks.get_mut(&key);
+			if let Some(_) = key_lock.and_then(Arc::get_mut) {
+				locks.remove(&key);
+			}
+		}
+		is_set
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use client_api::OffchainStorage;
+
+	#[test]
+	fn should_compare_and_set_and_clear_the_locks_map() {
+		let mut storage = LocalStorage::new_test();
+		let prefix = b"prefix";
+		let key = b"key";
+		let value = b"value";
+
+		storage.set(prefix, key, value);
+		assert_eq!(storage.get(prefix, key), Some(value.to_vec()));
+
+		assert_eq!(storage.compare_and_set(prefix, key, Some(value), b"asd"), true);
+		assert_eq!(storage.get(prefix, key), Some(b"asd".to_vec()));
+		assert!(storage.locks.lock().is_empty(), "Locks map should be empty!");
+	}
+
+	#[test]
+	fn should_compare_and_set_on_empty_field() {
+		let mut storage = LocalStorage::new_test();
+		let prefix = b"prefix";
+		let key = b"key";
+
+		assert_eq!(storage.compare_and_set(prefix, key, None, b"asd"), true);
+		assert_eq!(storage.get(prefix, key), Some(b"asd".to_vec()));
+		assert!(storage.locks.lock().is_empty(), "Locks map should be empty!");
+	}
+
+}

substrate/client/db/src/utils.rs

@@ -0,0 +1,356 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Db-based backend utility structures and functions, used by both
+//! full and light storages.
+
+use std::sync::Arc;
+use std::{io, convert::TryInto};
+
+use kvdb::{KeyValueDB, DBTransaction};
+#[cfg(feature = "kvdb-rocksdb")]
+use kvdb_rocksdb::{Database, DatabaseConfig};
+use log::debug;
+
+use client;
+use codec::Decode;
+use trie::DBValue;
+use sr_primitives::generic::BlockId;
+use sr_primitives::traits::{
+	Block as BlockT, Header as HeaderT, Zero,
+	UniqueSaturatedFrom, UniqueSaturatedInto,
+};
+use crate::{DatabaseSettings, DatabaseSettingsSrc};
+
+/// Number of columns in the db. Must be the same for both full && light dbs.
+/// Otherwise RocksDb will fail to open database && check its type.
+pub const NUM_COLUMNS: u32 = 10;
+/// Meta column. The set of keys in the column is shared by full && light storages.
+pub const COLUMN_META: Option<u32> = Some(0);
+
+/// Keys of entries in COLUMN_META.
+pub mod meta_keys {
+	/// Type of storage (full or light).
+	pub const TYPE: &[u8; 4] = b"type";
+	/// Best block key.
+	pub const BEST_BLOCK: &[u8; 4] = b"best";
+	/// Last finalized block key.
+	pub const FINALIZED_BLOCK: &[u8; 5] = b"final";
+	/// Meta information prefix for list-based caches.
+	pub const CACHE_META_PREFIX: &[u8; 5] = b"cache";
+	/// Genesis block hash.
+	pub const GENESIS_HASH: &[u8; 3] = b"gen";
+	/// Leaves prefix list key.
+	pub const LEAF_PREFIX: &[u8; 4] = b"leaf";
+	/// Children prefix list key.
+	pub const CHILDREN_PREFIX: &[u8; 8] = b"children";
+}
+
+/// Database metadata.
+#[derive(Debug)]
+pub struct Meta<N, H> {
+	/// Hash of the best known block.
+	pub best_hash: H,
+	/// Number of the best known block.
+	pub best_number: N,
+	/// Hash of the best finalized block.
+	pub finalized_hash: H,
+	/// Number of the best finalized block.
+	pub finalized_number: N,
+	/// Hash of the genesis block.
+	pub genesis_hash: H,
+}
+
+/// A block lookup key: used for canonical lookup from block number to hash
+pub type NumberIndexKey = [u8; 4];
+
+/// Convert block number into short lookup key (LE representation) for
+/// blocks that are in the canonical chain.
+///
+/// In the current database schema, this kind of key is only used for
+/// lookups into an index, NOT for storing header data or others.
+pub fn number_index_key<N: TryInto<u32>>(n: N) -> client::error::Result<NumberIndexKey> {
+	let n = n.try_into().map_err(|_|
+		client::error::Error::Backend("Block number cannot be converted to u32".into())
+	)?;
+
+	Ok([
+		(n >> 24) as u8,
+		((n >> 16) & 0xff) as u8,
+		((n >> 8) & 0xff) as u8,
+		(n & 0xff) as u8
+	])
+}
+
+/// Convert number and hash into long lookup key for blocks that are
+/// not in the canonical chain.
+pub fn number_and_hash_to_lookup_key<N, H>(
+	number: N,
+	hash: H,
+) -> client::error::Result<Vec<u8>>	where
+	N: TryInto<u32>,
+	H: AsRef<[u8]>,
+{
+	let mut lookup_key = number_index_key(number)?.to_vec();
+	lookup_key.extend_from_slice(hash.as_ref());
+	Ok(lookup_key)
+}
+
+/// Convert block lookup key into block number.
+/// all block lookup keys start with the block number.
+pub fn lookup_key_to_number<N>(key: &[u8]) -> client::error::Result<N> where
+	N: From<u32>
+{
+	if key.len() < 4 {
+		return Err(client::error::Error::Backend("Invalid block key".into()));
+	}
+	Ok((key[0] as u32) << 24
+		| (key[1] as u32) << 16
+		| (key[2] as u32) << 8
+		| (key[3] as u32)).map(Into::into)
+}
+
+/// Delete number to hash mapping in DB transaction.
+pub fn remove_number_to_key_mapping<N: TryInto<u32>>(
+	transaction: &mut DBTransaction,
+	key_lookup_col: Option<u32>,
+	number: N,
+) -> client::error::Result<()> {
+	transaction.delete(key_lookup_col, number_index_key(number)?.as_ref());
+	Ok(())
+}
+
+/// Remove key mappings.
+pub fn remove_key_mappings<N: TryInto<u32>, H: AsRef<[u8]>>(
+	transaction: &mut DBTransaction,
+	key_lookup_col: Option<u32>,
+	number: N,
+	hash: H,
+) -> client::error::Result<()> {
+	remove_number_to_key_mapping(transaction, key_lookup_col, number)?;
+	transaction.delete(key_lookup_col, hash.as_ref());
+	Ok(())
+}
+
+/// Place a number mapping into the database. This maps number to current perceived
+/// block hash at that position.
+pub fn insert_number_to_key_mapping<N: TryInto<u32> + Clone, H: AsRef<[u8]>>(
+	transaction: &mut DBTransaction,
+	key_lookup_col: Option<u32>,
+	number: N,
+	hash: H,
+) -> client::error::Result<()> {
+	transaction.put_vec(
+		key_lookup_col,
+		number_index_key(number.clone())?.as_ref(),
+		number_and_hash_to_lookup_key(number, hash)?,
+	);
+	Ok(())
+}
+
+/// Insert a hash to key mapping in the database.
+pub fn insert_hash_to_key_mapping<N: TryInto<u32>, H: AsRef<[u8]> + Clone>(
+	transaction: &mut DBTransaction,
+	key_lookup_col: Option<u32>,
+	number: N,
+	hash: H,
+) -> client::error::Result<()> {
+	transaction.put_vec(
+		key_lookup_col,
+		hash.clone().as_ref(),
+		number_and_hash_to_lookup_key(number, hash)?,
+	);
+	Ok(())
+}
+
+/// Convert block id to block lookup key.
+/// block lookup key is the DB-key header, block and justification are stored under.
+/// looks up lookup key by hash from DB as necessary.
+pub fn block_id_to_lookup_key<Block>(
+	db: &dyn KeyValueDB,
+	key_lookup_col: Option<u32>,
+	id: BlockId<Block>
+) -> Result<Option<Vec<u8>>, client::error::Error> where
+	Block: BlockT,
+	::sr_primitives::traits::NumberFor<Block>: UniqueSaturatedFrom<u64> + UniqueSaturatedInto<u64>,
+{
+	let res = match id {
+		BlockId::Number(n) => db.get(
+			key_lookup_col,
+			number_index_key(n)?.as_ref(),
+		),
+		BlockId::Hash(h) => db.get(key_lookup_col, h.as_ref()),
+	};
+
+	res.map(|v| v.map(|v| v.into_vec())).map_err(db_err)
+}
+
+/// Maps database error to client error
+pub fn db_err(err: io::Error) -> client::error::Error {
+	client::error::Error::Backend(format!("{}", err))
+}
+
+/// Open RocksDB database.
+pub fn open_database(
+	config: &DatabaseSettings,
+	col_meta: Option<u32>,
+	db_type: &str
+) -> client::error::Result<Arc<dyn KeyValueDB>> {
+	let db: Arc<dyn KeyValueDB> = match &config.source {
+		#[cfg(feature = "kvdb-rocksdb")]
+		DatabaseSettingsSrc::Path { path, cache_size } => {
+			let mut db_config = DatabaseConfig::with_columns(Some(NUM_COLUMNS));
+			db_config.memory_budget = *cache_size;
+			let path = path.to_str().ok_or_else(|| client::error::Error::Backend("Invalid database path".into()))?;
+			Arc::new(Database::open(&db_config, &path).map_err(db_err)?)
+		},
+		#[cfg(not(feature = "kvdb-rocksdb"))]
+		DatabaseSettingsSrc::Path { .. } => {
+			let msg = "Try to open RocksDB database with RocksDB disabled".into();
+			return Err(client::error::Error::Backend(msg));
+		},
+		DatabaseSettingsSrc::Custom(db) => db.clone(),
+	};
+
+	// check database type
+	match db.get(col_meta, meta_keys::TYPE).map_err(db_err)? {
+		Some(stored_type) => {
+			if db_type.as_bytes() != &*stored_type {
+				return Err(client::error::Error::Backend(
+					format!("Unexpected database type. Expected: {}", db_type)).into());
+			}
+		},
+		None => {
+			let mut transaction = DBTransaction::new();
+			transaction.put(col_meta, meta_keys::TYPE, db_type.as_bytes());
+			db.write(transaction).map_err(db_err)?;
+		},
+	}
+
+	Ok(db)
+}
+
+/// Read database column entry for the given block.
+pub fn read_db<Block>(
+	db: &dyn KeyValueDB,
+	col_index: Option<u32>,
+	col: Option<u32>,
+	id: BlockId<Block>
+) -> client::error::Result<Option<DBValue>>
+	where
+		Block: BlockT,
+{
+	block_id_to_lookup_key(db, col_index, id).and_then(|key| match key {
+		Some(key) => db.get(col, key.as_ref()).map_err(db_err),
+		None => Ok(None),
+	})
+}
+
+/// Read a header from the database.
+pub fn read_header<Block: BlockT>(
+	db: &dyn KeyValueDB,
+	col_index: Option<u32>,
+	col: Option<u32>,
+	id: BlockId<Block>,
+) -> client::error::Result<Option<Block::Header>> {
+	match read_db(db, col_index, col, id)? {
+		Some(header) => match Block::Header::decode(&mut &header[..]) {
+			Ok(header) => Ok(Some(header)),
+			Err(_) => return Err(
+				client::error::Error::Backend("Error decoding header".into())
+			),
+		}
+		None => Ok(None),
+	}
+}
+
+/// Required header from the database.
+pub fn require_header<Block: BlockT>(
+	db: &dyn KeyValueDB,
+	col_index: Option<u32>,
+	col: Option<u32>,
+	id: BlockId<Block>,
+) -> client::error::Result<Block::Header> {
+	read_header(db, col_index, col, id)
+		.and_then(|header| header.ok_or_else(|| client::error::Error::UnknownBlock(format!("{}", id))))
+}
+
+/// Read meta from the database.
+pub fn read_meta<Block>(db: &dyn KeyValueDB, col_meta: Option<u32>, col_header: Option<u32>) -> Result<
+	Meta<<<Block as BlockT>::Header as HeaderT>::Number, Block::Hash>,
+	client::error::Error,
+>
+	where
+		Block: BlockT,
+{
+	let genesis_hash: Block::Hash = match db.get(col_meta, meta_keys::GENESIS_HASH).map_err(db_err)? {
+		Some(h) => match Decode::decode(&mut &h[..]) {
+			Ok(h) => h,
+			Err(err) => return Err(client::error::Error::Backend(
+				format!("Error decoding genesis hash: {}", err)
+			)),
+		},
+		None => return Ok(Meta {
+			best_hash: Default::default(),
+			best_number: Zero::zero(),
+			finalized_hash: Default::default(),
+			finalized_number: Zero::zero(),
+			genesis_hash: Default::default(),
+		}),
+	};
+
+	let load_meta_block = |desc, key| -> Result<_, client::error::Error> {
+		if let Some(Some(header)) = db.get(col_meta, key).and_then(|id|
+			match id {
+				Some(id) => db.get(col_header, &id).map(|h| h.map(|b| Block::Header::decode(&mut &b[..]).ok())),
+				None => Ok(None),
+			}).map_err(db_err)?
+		{
+			let hash = header.hash();
+			debug!("DB Opened blockchain db, fetched {} = {:?} ({})", desc, hash, header.number());
+			Ok((hash, *header.number()))
+		} else {
+			Ok((genesis_hash.clone(), Zero::zero()))
+		}
+	};
+
+	let (best_hash, best_number) = load_meta_block("best", meta_keys::BEST_BLOCK)?;
+	let (finalized_hash, finalized_number) = load_meta_block("final", meta_keys::FINALIZED_BLOCK)?;
+
+	Ok(Meta {
+		best_hash,
+		best_number,
+		finalized_hash,
+		finalized_number,
+		genesis_hash,
+	})
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use sr_primitives::testing::{Block as RawBlock, ExtrinsicWrapper};
+	type Block = RawBlock<ExtrinsicWrapper<u32>>;
+
+	#[test]
+	fn number_index_key_doesnt_panic() {
+		let id = BlockId::<Block>::Number(72340207214430721);
+		match id {
+			BlockId::Number(n) => number_index_key(n).expect_err("number should overflow u32"),
+			_ => unreachable!(),
+		};
+	}
+}

substrate/client/executor/Cargo.toml

@@ -0,0 +1,60 @@
+[package]
+name = "substrate-executor"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+derive_more = "0.15.0"
+codec = { package = "parity-scale-codec", version = "1.0.0" }
+runtime_io = { package = "sr-io", path = "../../primitives/sr-io" }
+primitives = { package = "substrate-primitives", path = "../../primitives/core" }
+trie = { package = "substrate-trie", path = "../../primitives/trie" }
+serializer = { package = "substrate-serializer", path = "../../primitives/serializer" }
+runtime_version = { package = "sr-version", path = "../../primitives/sr-version" }
+panic-handler = { package = "substrate-panic-handler", path = "../../primitives/panic-handler" }
+wasmi = "0.6.2"
+parity-wasm = "0.40.3"
+lazy_static = "1.4.0"
+wasm-interface = { package = "substrate-wasm-interface", path = "../../primitives/wasm-interface" }
+runtime-interface = { package = "substrate-runtime-interface", path = "../../primitives/runtime-interface" }
+externalities = { package = "substrate-externalities", path = "../../primitives/externalities" }
+parking_lot = "0.9.0"
+log = "0.4.8"
+libsecp256k1 = "0.3.2"
+
+cranelift-codegen = { version = "0.46.1", optional = true }
+cranelift-entity = { version = "0.46.1", optional = true }
+cranelift-frontend = { version = "0.46.1", optional = true }
+cranelift-native = { version = "0.46.1", optional = true }
+cranelift-wasm = { version = "0.46.1", optional = true }
+wasmtime-environ = { version = "0.2", optional = true, git = "https://github.com/CraneStation/wasmtime.git", rev = "71dd73d6" }
+wasmtime-jit = { version = "0.2", optional = true, git = "https://github.com/CraneStation/wasmtime.git", rev = "71dd73d6" }
+wasmtime-runtime = { version = "0.2", optional = true, git = "https://github.com/CraneStation/wasmtime.git", rev = "71dd73d6" }
+
+[dev-dependencies]
+assert_matches = "1.3.0"
+wabt = "0.9.2"
+hex-literal = "0.2.1"
+runtime-test = { package = "substrate-runtime-test", path = "runtime-test" }
+client-api = { package = "substrate-client-api", path = "../api" }
+substrate-offchain = { path = "../offchain/" }
+state_machine = { package = "substrate-state-machine", path = "../../primitives/state-machine"  }
+test-case = "0.3.3"
+
+[features]
+default = []
+wasm-extern-trace = []
+wasmtime = [
+	"cranelift-codegen",
+	"cranelift-entity",
+	"cranelift-frontend",
+	"cranelift-native",
+	"cranelift-wasm",
+	"wasmtime-environ",
+	"wasmtime-jit",
+	"wasmtime-runtime",
+]
+wasmi-errno = [
+	"wasmi/errno"
+]

substrate/client/executor/runtime-test/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "substrate-runtime-test"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+build = "build.rs"
+
+[dependencies]
+rstd = { package = "sr-std", path = "../../../primitives/sr-std", default-features = false }
+runtime_io = { package = "sr-io", path = "../../../primitives/sr-io", default-features = false }
+sandbox = { package = "sr-sandbox", path = "../../../primitives/sr-sandbox", default-features = false }
+primitives = { package = "substrate-primitives",  path = "../../../primitives/core", default-features = false }
+sr-primitives = { package = "sr-primitives",  path = "../../../primitives/sr-primitives", default-features = false }
+
+[build-dependencies]
+wasm-builder-runner = { package = "substrate-wasm-builder-runner", path = "../../../client/utils/wasm-builder-runner", version = "1.0.4" }
+
+[features]
+default = [ "std" ]
+std = ["runtime_io/std", "sandbox/std", "rstd/std"]

substrate/client/executor/runtime-test/build.rs

@@ -0,0 +1,30 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use wasm_builder_runner::{build_current_project_with_rustflags, WasmBuilderSource};
+
+fn main() {
+	build_current_project_with_rustflags(
+		"wasm_binary.rs",
+		WasmBuilderSource::CratesOrPath {
+			path: "../../../utils/wasm-builder",
+			version: "1.0.8",
+		},
+		// This instructs LLD to export __heap_base as a global variable, which is used by the
+		// external memory allocator.
+		"-Clink-arg=--export=__heap_base",
+	);
+}

substrate/client/executor/runtime-test/src/lib.rs

@@ -0,0 +1,254 @@
+#![cfg_attr(not(feature = "std"), no_std)]
+#![cfg_attr(feature = "strict", deny(warnings))]
+
+// Make the WASM binary available.
+#[cfg(feature = "std")]
+include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
+
+#[cfg(not(feature = "std"))]
+use rstd::{vec::Vec, vec};
+
+#[cfg(not(feature = "std"))]
+use runtime_io::{
+	storage, hashing::{blake2_128, blake2_256, twox_128, twox_256},
+	crypto::{ed25519_verify, sr25519_verify},
+};
+#[cfg(not(feature = "std"))]
+use sr_primitives::{print, traits::{BlakeTwo256, Hash}};
+#[cfg(not(feature = "std"))]
+use primitives::{ed25519, sr25519};
+
+primitives::wasm_export_functions! {
+	fn test_data_in(input: Vec<u8>) -> Vec<u8> {
+		print("set_storage");
+		storage::set(b"input", &input);
+
+		print("storage");
+		let foo = storage::get(b"foo").unwrap();
+
+		print("set_storage");
+		storage::set(b"baz", &foo);
+
+		print("finished!");
+		b"all ok!".to_vec()
+	}
+
+	fn test_clear_prefix(input: Vec<u8>) -> Vec<u8> {
+		storage::clear_prefix(&input);
+		b"all ok!".to_vec()
+	}
+
+	fn test_empty_return() {}
+
+	fn test_exhaust_heap() -> Vec<u8> { Vec::with_capacity(16777216) }
+
+	fn test_panic() { panic!("test panic") }
+
+	fn test_conditional_panic(input: Vec<u8>) -> Vec<u8> {
+		if input.len() > 0 {
+			panic!("test panic")
+		}
+
+		input
+	}
+
+	fn test_blake2_256(input: Vec<u8>) -> Vec<u8> {
+		blake2_256(&input).to_vec()
+	}
+
+	fn test_blake2_128(input: Vec<u8>) -> Vec<u8> {
+		blake2_128(&input).to_vec()
+	}
+
+	fn test_twox_256(input: Vec<u8>) -> Vec<u8> {
+		twox_256(&input).to_vec()
+	}
+
+	fn test_twox_128(input: Vec<u8>) -> Vec<u8> {
+		twox_128(&input).to_vec()
+	}
+
+	fn test_ed25519_verify(input: Vec<u8>) -> bool {
+		let mut pubkey = [0; 32];
+		let mut sig = [0; 64];
+
+		pubkey.copy_from_slice(&input[0..32]);
+		sig.copy_from_slice(&input[32..96]);
+
+		let msg = b"all ok!";
+		ed25519_verify(&ed25519::Signature(sig), &msg[..], &ed25519::Public(pubkey))
+	}
+
+	fn test_sr25519_verify(input: Vec<u8>) -> bool {
+		let mut pubkey = [0; 32];
+		let mut sig = [0; 64];
+
+		pubkey.copy_from_slice(&input[0..32]);
+		sig.copy_from_slice(&input[32..96]);
+
+		let msg = b"all ok!";
+		sr25519_verify(&sr25519::Signature(sig), &msg[..], &sr25519::Public(pubkey))
+	}
+
+	fn test_ordered_trie_root() -> Vec<u8> {
+		BlakeTwo256::ordered_trie_root(
+			vec![
+				b"zero"[..].into(),
+				b"one"[..].into(),
+				b"two"[..].into(),
+			],
+		).as_ref().to_vec()
+	}
+
+	fn test_sandbox(code: Vec<u8>) -> bool {
+		execute_sandboxed(&code, &[]).is_ok()
+	}
+
+	fn test_sandbox_args(code: Vec<u8>) -> bool {
+		execute_sandboxed(
+			&code,
+			&[
+				sandbox::TypedValue::I32(0x12345678),
+				sandbox::TypedValue::I64(0x1234567887654321),
+			],
+		).is_ok()
+	}
+
+	fn test_sandbox_return_val(code: Vec<u8>) -> bool {
+		let ok = match execute_sandboxed(
+			&code,
+			&[
+				sandbox::TypedValue::I32(0x1336),
+			]
+		) {
+			Ok(sandbox::ReturnValue::Value(sandbox::TypedValue::I32(0x1337))) => true,
+			_ => false,
+		};
+
+		ok
+	}
+
+	fn test_sandbox_instantiate(code: Vec<u8>) -> u8 {
+		let env_builder = sandbox::EnvironmentDefinitionBuilder::new();
+		let code = match sandbox::Instance::new(&code, &env_builder, &mut ()) {
+			Ok(_) => 0,
+			Err(sandbox::Error::Module) => 1,
+			Err(sandbox::Error::Execution) => 2,
+			Err(sandbox::Error::OutOfBounds) => 3,
+		};
+
+		code
+	}
+
+	fn test_offchain_local_storage() -> bool {
+		let kind = primitives::offchain::StorageKind::PERSISTENT;
+		assert_eq!(runtime_io::offchain::local_storage_get(kind, b"test"), None);
+		runtime_io::offchain::local_storage_set(kind, b"test", b"asd");
+		assert_eq!(runtime_io::offchain::local_storage_get(kind, b"test"), Some(b"asd".to_vec()));
+
+		let res = runtime_io::offchain::local_storage_compare_and_set(
+			kind,
+			b"test",
+			Some(b"asd".to_vec()),
+			b"",
+		);
+		assert_eq!(runtime_io::offchain::local_storage_get(kind, b"test"), Some(b"".to_vec()));
+		res
+	}
+
+	fn test_offchain_local_storage_with_none() {
+		let kind = primitives::offchain::StorageKind::PERSISTENT;
+		assert_eq!(runtime_io::offchain::local_storage_get(kind, b"test"), None);
+
+		let res = runtime_io::offchain::local_storage_compare_and_set(kind, b"test", None, b"value");
+		assert_eq!(res, true);
+		assert_eq!(runtime_io::offchain::local_storage_get(kind, b"test"), Some(b"value".to_vec()));
+	}
+
+	fn test_offchain_http() -> bool {
+		use primitives::offchain::HttpRequestStatus;
+		let run = || -> Option<()> {
+			let id = runtime_io::offchain::http_request_start(
+				"POST",
+				"http://localhost:12345",
+				&[],
+			).ok()?;
+			runtime_io::offchain::http_request_add_header(id, "X-Auth", "test").ok()?;
+			runtime_io::offchain::http_request_write_body(id, &[1, 2, 3, 4], None).ok()?;
+			runtime_io::offchain::http_request_write_body(id, &[], None).ok()?;
+			let status = runtime_io::offchain::http_response_wait(&[id], None);
+			assert!(status == vec![HttpRequestStatus::Finished(200)], "Expected Finished(200) status.");
+			let headers = runtime_io::offchain::http_response_headers(id);
+			assert_eq!(headers, vec![(b"X-Auth".to_vec(), b"hello".to_vec())]);
+			let mut buffer = vec![0; 64];
+			let read = runtime_io::offchain::http_response_read_body(id, &mut buffer, None).ok()?;
+			assert_eq!(read, 3);
+			assert_eq!(&buffer[0..read as usize], &[1, 2, 3]);
+			let read = runtime_io::offchain::http_response_read_body(id, &mut buffer, None).ok()?;
+			assert_eq!(read, 0);
+
+			Some(())
+		};
+
+		run().is_some()
+	}
+ }
+
+#[cfg(not(feature = "std"))]
+fn execute_sandboxed(
+	code: &[u8],
+	args: &[sandbox::TypedValue],
+) -> Result<sandbox::ReturnValue, sandbox::HostError> {
+	struct State {
+		counter: u32,
+	}
+
+	fn env_assert(
+		_e: &mut State,
+		args: &[sandbox::TypedValue],
+	) -> Result<sandbox::ReturnValue, sandbox::HostError> {
+		if args.len() != 1 {
+			return Err(sandbox::HostError);
+		}
+		let condition = args[0].as_i32().ok_or_else(|| sandbox::HostError)?;
+		if condition != 0 {
+			Ok(sandbox::ReturnValue::Unit)
+		} else {
+			Err(sandbox::HostError)
+		}
+	}
+	fn env_inc_counter(
+		e: &mut State,
+		args: &[sandbox::TypedValue],
+	) -> Result<sandbox::ReturnValue, sandbox::HostError> {
+		if args.len() != 1 {
+			return Err(sandbox::HostError);
+		}
+		let inc_by = args[0].as_i32().ok_or_else(|| sandbox::HostError)?;
+		e.counter += inc_by as u32;
+		Ok(sandbox::ReturnValue::Value(sandbox::TypedValue::I32(e.counter as i32)))
+	}
+
+	let mut state = State { counter: 0 };
+
+	let env_builder = {
+		let mut env_builder = sandbox::EnvironmentDefinitionBuilder::new();
+		env_builder.add_host_func("env", "assert", env_assert);
+		env_builder.add_host_func("env", "inc_counter", env_inc_counter);
+		let memory = match sandbox::Memory::new(1, Some(16)) {
+			Ok(m) => m,
+			Err(_) => unreachable!("
+				Memory::new() can return Err only if parameters are borked; \
+				We passing params here explicitly and they're correct; \
+				Memory::new() can't return a Error qed"
+			),
+		};
+		env_builder.add_memory("env", "memory", memory.clone());
+		env_builder
+	};
+
+	let mut instance = sandbox::Instance::new(code, &env_builder, &mut state)?;
+	let result = instance.invoke("call", args, &mut state);
+
+	result.map_err(|_| sandbox::HostError)
+}

substrate/client/executor/src/allocator.rs

@@ -0,0 +1,554 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! This module implements a freeing-bump allocator.
+//!
+//! The algorithm is as follows:
+//! We store `N` linked list heads, where `N` is the total number of sizes
+//! of allocations to support. A simple set is powers of two from 8 bytes
+//! to 16,777,216 bytes (2^3 - 2^24 inclusive), resulting in `N = 22`:
+//!
+//! ```ignore
+//!	let mut heads [u64; N] = [0; N];
+//! fn size(n: u64) -> u64 { 8 << n }
+//! let mut bumper = 0;
+//! fn bump(n: u64) -> u64 { let res = bumper; bumper += n; res }
+//! ```
+//!
+//! We assume there is a slab of heap to be allocated:
+//!
+//! ```ignore
+//! let mut heap = [0u8; HEAP_SIZE];
+//! ```
+//!
+//! Whenever we allocate, we select the lowest linked list item size that
+//! will fit the allocation (i.e. the next highest power of two).
+//! We then check to see if the linked list is empty. If empty, we use
+//! the bump allocator to get the allocation with an extra 8 bytes
+//! preceding it. We initialise those preceding 8 bytes to identify the
+//! list to which it belongs. If it is not empty, we unlink the first item from
+//! the linked list and then reset the 8 preceding bytes so they now record
+//! the identity of the linked list.
+//!
+//! To deallocate we use the preceding 8 bytes of the allocation to knit
+//! back the allocation into the linked list from the head.
+
+use crate::error::{Error, Result};
+use log::trace;
+use std::convert::{TryFrom, TryInto};
+use std::ops::Range;
+use wasm_interface::{Pointer, WordSize};
+
+// The pointers need to be aligned to 8 bytes. This is because the
+// maximum value type handled by wasm32 is u64.
+const ALIGNMENT: u32 = 8;
+
+// The pointer returned by `allocate()` needs to fulfill the alignment
+// requirement. In our case a pointer will always be a multiple of
+// 8, as long as the first pointer is aligned to 8 bytes.
+// This is because all pointers will contain a 8 byte prefix (the list
+// index) and then a subsequent item of 2^x bytes, where x = [3..24].
+const N: usize = 22;
+const MAX_POSSIBLE_ALLOCATION: u32 = 16777216; // 2^24 bytes
+const MIN_POSSIBLE_ALLOCATION: u32 = 8;
+
+// Each pointer is prefixed with 8 bytes, which identify the list index
+// to which it belongs.
+const PREFIX_SIZE: u32 = 8;
+
+pub struct FreeingBumpHeapAllocator {
+	bumper: u32,
+	heads: [u32; N],
+	ptr_offset: u32,
+	total_size: u32,
+}
+
+/// Create an allocator error.
+fn error(msg: &'static str) -> Error {
+	Error::Allocator(msg)
+}
+
+impl FreeingBumpHeapAllocator {
+	/// Creates a new allocation heap which follows a freeing-bump strategy.
+	/// The maximum size which can be allocated at once is 16 MiB.
+	///
+	/// # Arguments
+	///
+	/// - `heap_base` - the offset from the beginning of the linear memory where the heap starts.
+	pub fn new(heap_base: u32) -> Self {
+		// ptr_offset is the next alignment boundary on or after heap_base.
+		let ptr_offset = (heap_base + ALIGNMENT - 1) / ALIGNMENT * ALIGNMENT;
+
+		FreeingBumpHeapAllocator {
+			bumper: 0,
+			heads: [0; N],
+			ptr_offset,
+			total_size: 0,
+		}
+	}
+
+	/// Gets requested number of bytes to allocate and returns a pointer.
+	/// The maximum size which can be allocated at once is 16 MiB.
+	/// There is no minimum size, but whatever size is passed into
+	/// this function is rounded to the next power of two. If the requested
+	/// size is below 8 bytes it will be rounded up to 8 bytes.
+	///
+	/// # Arguments
+	///
+	/// - `mem` - a slice representing the linear memory on which this allocator operates.
+	/// - `size` - size in bytes of the allocation request
+	pub fn allocate(&mut self, mem: &mut [u8], size: WordSize) -> Result<Pointer<u8>> {
+		let mem_size = u32::try_from(mem.len())
+			.expect("size of Wasm linear memory is <2^32");
+		let max_heap_size = mem_size - self.ptr_offset;
+
+		if size > MAX_POSSIBLE_ALLOCATION {
+			return Err(Error::RequestedAllocationTooLarge);
+		}
+
+		let size = size.max(MIN_POSSIBLE_ALLOCATION);
+		let item_size = size.next_power_of_two();
+		if item_size + PREFIX_SIZE + self.total_size > max_heap_size {
+			return Err(Error::AllocatorOutOfSpace);
+		}
+
+		let list_index = (item_size.trailing_zeros() - 3) as usize;
+		let ptr: u32 = if self.heads[list_index] != 0 {
+			// Something from the free list
+			let item = self.heads[list_index];
+			let ptr = item + PREFIX_SIZE;
+			assert!(
+				ptr + item_size <= max_heap_size,
+				"Pointer is looked up in list of free entries, into which
+				only valid values are inserted; qed"
+			);
+
+			self.heads[list_index] = self.get_heap_u64(mem, item)?
+				.try_into()
+				.map_err(|_| error("read invalid free list pointer"))?;
+			ptr
+		} else {
+			// Nothing to be freed. Bump.
+			self.bump(item_size, max_heap_size)? + PREFIX_SIZE
+		};
+
+		self.set_heap_u64(mem, ptr - PREFIX_SIZE, list_index as u64)?;
+
+		self.total_size = self.total_size + item_size + PREFIX_SIZE;
+		trace!(target: "wasm-heap", "Heap size is {} bytes after allocation", self.total_size);
+
+		Ok(Pointer::new(self.ptr_offset + ptr))
+	}
+
+	/// Deallocates the space which was allocated for a pointer.
+	///
+	/// # Arguments
+	///
+	/// - `mem` - a slice representing the linear memory on which this allocator operates.
+	/// - `ptr` - pointer to the allocated chunk
+	pub fn deallocate(&mut self, mem: &mut [u8], ptr: Pointer<u8>) -> Result<()> {
+		let ptr = u32::from(ptr) - self.ptr_offset;
+		if ptr < PREFIX_SIZE {
+			return Err(error("Invalid pointer for deallocation"));
+		}
+
+		let list_index: usize = self.get_heap_u64(mem, ptr - PREFIX_SIZE)?
+			.try_into()
+			.map_err(|_| error("read invalid list index"))?;
+		if list_index > self.heads.len() {
+			return Err(error("read invalid list index"));
+		}
+		self.set_heap_u64(mem, ptr - PREFIX_SIZE, self.heads[list_index] as u64)?;
+		self.heads[list_index] = ptr - PREFIX_SIZE;
+
+		let item_size = Self::get_item_size_from_index(list_index);
+		self.total_size = self.total_size.checked_sub(item_size as u32 + PREFIX_SIZE)
+			.ok_or_else(|| error("Unable to subtract from total heap size without overflow"))?;
+		trace!(target: "wasm-heap", "Heap size is {} bytes after deallocation", self.total_size);
+
+		Ok(())
+	}
+
+	/// Increases the `bumper` by `item_size + PREFIX_SIZE`.
+	///
+	/// Returns the `bumper` from before the increase.
+	/// Returns an `Error::AllocatorOutOfSpace` if the operation
+	/// would exhaust the heap.
+	fn bump(&mut self, item_size: u32, max_heap_size: u32) -> Result<u32> {
+		if self.bumper + PREFIX_SIZE + item_size > max_heap_size {
+			return Err(Error::AllocatorOutOfSpace);
+		}
+
+		let res = self.bumper;
+		self.bumper += item_size + PREFIX_SIZE;
+		Ok(res)
+	}
+
+	fn get_item_size_from_index(index: usize) -> usize {
+		// we shift 1 by three places, since the first possible item size is 8
+		1 << 3 << index
+	}
+
+	// Read a u64 from the heap in LE form. Used to read heap allocation prefixes.
+	fn get_heap_u64(&self, heap: &[u8], offset: u32) -> Result<u64> {
+		let range = self.heap_range(offset, 8, heap.len())
+			.ok_or_else(|| error("read out of heap bounds"))?;
+		let bytes = heap[range].try_into()
+			.expect("[u8] slice of length 8 must be convertible to [u8; 8]");
+		Ok(u64::from_le_bytes(bytes))
+	}
+
+	// Write a u64 to the heap in LE form. Used to write heap allocation prefixes.
+	fn set_heap_u64(&self, heap: &mut [u8], offset: u32, val: u64) -> Result<()> {
+		let range = self.heap_range(offset, 8, heap.len())
+			.ok_or_else(|| error("write out of heap bounds"))?;
+		let bytes = val.to_le_bytes();
+		&mut heap[range].copy_from_slice(&bytes[..]);
+		Ok(())
+	}
+
+	fn heap_range(&self, offset: u32, length: u32, heap_len: usize) -> Option<Range<usize>> {
+		let start = offset
+			.checked_add(self.ptr_offset)?
+			as usize;
+		let end = offset
+			.checked_add(self.ptr_offset)?
+			.checked_add(length)?
+			as usize;
+		if end <= heap_len {
+			Some(start..end)
+		} else {
+			None
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	const PAGE_SIZE: u32 = 65536;
+
+	/// Makes a pointer out of the given address.
+	fn to_pointer(address: u32) -> Pointer<u8> {
+		Pointer::new(address)
+	}
+
+	#[test]
+	fn should_allocate_properly() {
+		// given
+		let mut mem = [0u8; PAGE_SIZE as usize];
+		let mut heap = FreeingBumpHeapAllocator::new(0);
+
+		// when
+		let ptr = heap.allocate(&mut mem[..], 1).unwrap();
+
+		// then
+		// returned pointer must start right after `PREFIX_SIZE`
+		assert_eq!(ptr, to_pointer(PREFIX_SIZE));
+	}
+
+	#[test]
+	fn should_always_align_pointers_to_multiples_of_8() {
+		// given
+		let mut mem = [0u8; PAGE_SIZE as usize];
+		let mut heap = FreeingBumpHeapAllocator::new(13);
+
+		// when
+		let ptr = heap.allocate(&mut mem[..], 1).unwrap();
+
+		// then
+		// the pointer must start at the next multiple of 8 from 13
+		// + the prefix of 8 bytes.
+		assert_eq!(ptr, to_pointer(24));
+	}
+
+	#[test]
+	fn should_increment_pointers_properly() {
+		// given
+		let mut mem = [0u8; PAGE_SIZE as usize];
+		let mut heap = FreeingBumpHeapAllocator::new(0);
+
+		// when
+		let ptr1 = heap.allocate(&mut mem[..], 1).unwrap();
+		let ptr2 = heap.allocate(&mut mem[..], 9).unwrap();
+		let ptr3 = heap.allocate(&mut mem[..], 1).unwrap();
+
+		// then
+		// a prefix of 8 bytes is prepended to each pointer
+		assert_eq!(ptr1, to_pointer(PREFIX_SIZE));
+
+		// the prefix of 8 bytes + the content of ptr1 padded to the lowest possible
+		// item size of 8 bytes + the prefix of ptr1
+		assert_eq!(ptr2, to_pointer(24));
+
+		// ptr2 + its content of 16 bytes + the prefix of 8 bytes
+		assert_eq!(ptr3, to_pointer(24 + 16 + PREFIX_SIZE));
+	}
+
+	#[test]
+	fn should_free_properly() {
+		// given
+		let mut mem = [0u8; PAGE_SIZE as usize];
+		let mut heap = FreeingBumpHeapAllocator::new(0);
+		let ptr1 = heap.allocate(&mut mem[..], 1).unwrap();
+		// the prefix of 8 bytes is prepended to the pointer
+		assert_eq!(ptr1, to_pointer(PREFIX_SIZE));
+
+		let ptr2 = heap.allocate(&mut mem[..], 1).unwrap();
+		// the prefix of 8 bytes + the content of ptr 1 is prepended to the pointer
+		assert_eq!(ptr2, to_pointer(24));
+
+		// when
+		heap.deallocate(&mut mem[..], ptr2).unwrap();
+
+		// then
+		// then the heads table should contain a pointer to the
+		// prefix of ptr2 in the leftmost entry
+		assert_eq!(heap.heads[0], u32::from(ptr2) - PREFIX_SIZE);
+	}
+
+	#[test]
+	fn should_deallocate_and_reallocate_properly() {
+		// given
+		let mut mem = [0u8; PAGE_SIZE as usize];
+		let padded_offset = 16;
+		let mut heap = FreeingBumpHeapAllocator::new(13);
+
+		let ptr1 = heap.allocate(&mut mem[..], 1).unwrap();
+		// the prefix of 8 bytes is prepended to the pointer
+		assert_eq!(ptr1, to_pointer(padded_offset + PREFIX_SIZE));
+
+		let ptr2 = heap.allocate(&mut mem[..], 9).unwrap();
+		// the padded_offset + the previously allocated ptr (8 bytes prefix +
+		// 8 bytes content) + the prefix of 8 bytes which is prepended to the
+		// current pointer
+		assert_eq!(ptr2, to_pointer(padded_offset + 16 + PREFIX_SIZE));
+
+		// when
+		heap.deallocate(&mut mem[..], ptr2).unwrap();
+		let ptr3 = heap.allocate(&mut mem[..], 9).unwrap();
+
+		// then
+		// should have re-allocated
+		assert_eq!(ptr3, to_pointer(padded_offset + 16 + PREFIX_SIZE));
+		assert_eq!(heap.heads, [0; N]);
+	}
+
+	#[test]
+	fn should_build_linked_list_of_free_areas_properly() {
+		// given
+		let mut mem = [0u8; PAGE_SIZE as usize];
+		let mut heap = FreeingBumpHeapAllocator::new(0);
+
+		let ptr1 = heap.allocate(&mut mem[..], 8).unwrap();
+		let ptr2 = heap.allocate(&mut mem[..], 8).unwrap();
+		let ptr3 = heap.allocate(&mut mem[..], 8).unwrap();
+
+		// when
+		heap.deallocate(&mut mem[..], ptr1).unwrap();
+		heap.deallocate(&mut mem[..], ptr2).unwrap();
+		heap.deallocate(&mut mem[..], ptr3).unwrap();
+
+		// then
+		assert_eq!(heap.heads[0], u32::from(ptr3) - PREFIX_SIZE);
+
+		let ptr4 = heap.allocate(&mut mem[..], 8).unwrap();
+		assert_eq!(ptr4, ptr3);
+
+		assert_eq!(heap.heads[0], u32::from(ptr2) - PREFIX_SIZE);
+	}
+
+	#[test]
+	fn should_not_allocate_if_too_large() {
+		// given
+		let mut mem = [0u8; PAGE_SIZE as usize];
+		let mut heap = FreeingBumpHeapAllocator::new(13);
+
+		// when
+		let ptr = heap.allocate(&mut mem[..], PAGE_SIZE - 13);
+
+		// then
+		match ptr.unwrap_err() {
+			Error::AllocatorOutOfSpace => {},
+			e => panic!("Expected allocator out of space error, got: {:?}", e),
+		}
+	}
+
+	#[test]
+	fn should_not_allocate_if_full() {
+		// given
+		let mut mem = [0u8; PAGE_SIZE as usize];
+		let mut heap = FreeingBumpHeapAllocator::new(0);
+		let ptr1 = heap.allocate(&mut mem[..], (PAGE_SIZE / 2) - PREFIX_SIZE).unwrap();
+		assert_eq!(ptr1, to_pointer(PREFIX_SIZE));
+
+		// when
+		let ptr2 = heap.allocate(&mut mem[..], PAGE_SIZE / 2);
+
+		// then
+		// there is no room for another half page incl. its 8 byte prefix
+		match ptr2.unwrap_err() {
+			Error::AllocatorOutOfSpace => {},
+			e => panic!("Expected allocator out of space error, got: {:?}", e),
+		}
+	}
+
+	#[test]
+	fn should_allocate_max_possible_allocation_size() {
+		// given
+		let mut mem = vec![0u8; (MAX_POSSIBLE_ALLOCATION + PAGE_SIZE) as usize];
+		let mut heap = FreeingBumpHeapAllocator::new(0);
+
+		// when
+		let ptr = heap.allocate(&mut mem[..], MAX_POSSIBLE_ALLOCATION).unwrap();
+
+		// then
+		assert_eq!(ptr, to_pointer(PREFIX_SIZE));
+	}
+
+	#[test]
+	fn should_not_allocate_if_requested_size_too_large() {
+		// given
+		let mut mem = [0u8; PAGE_SIZE as usize];
+		let mut heap = FreeingBumpHeapAllocator::new(0);
+
+		// when
+		let ptr = heap.allocate(&mut mem[..], MAX_POSSIBLE_ALLOCATION + 1);
+
+		// then
+		match ptr.unwrap_err() {
+			Error::RequestedAllocationTooLarge => {},
+			e => panic!("Expected allocation size too large error, got: {:?}", e),
+		}
+	}
+
+	#[test]
+	fn should_return_error_when_bumper_greater_than_heap_size() {
+		// given
+		let mut mem = [0u8; 64];
+		let mut heap = FreeingBumpHeapAllocator::new(0);
+
+		let ptr1 = heap.allocate(&mut mem[..], 32).unwrap();
+		assert_eq!(ptr1, to_pointer(PREFIX_SIZE));
+		heap.deallocate(&mut mem[..], ptr1).expect("failed freeing ptr1");
+		assert_eq!(heap.total_size, 0);
+		assert_eq!(heap.bumper, 40);
+
+		let ptr2 = heap.allocate(&mut mem[..], 16).unwrap();
+		assert_eq!(ptr2, to_pointer(48));
+		heap.deallocate(&mut mem[..], ptr2).expect("failed freeing ptr2");
+		assert_eq!(heap.total_size, 0);
+		assert_eq!(heap.bumper, 64);
+
+		// when
+		// the `bumper` value is equal to `max_heap_size` here and any
+		// further allocation which would increment the bumper must fail.
+		// we try to allocate 8 bytes here, which will increment the
+		// bumper since no 8 byte item has been allocated+freed before.
+		let ptr = heap.allocate(&mut mem[..], 8);
+
+		// then
+		match ptr.unwrap_err() {
+			Error::AllocatorOutOfSpace => {},
+			e => panic!("Expected allocator out of space error, got: {:?}", e),
+		}
+	}
+
+	#[test]
+	fn should_include_prefixes_in_total_heap_size() {
+		// given
+		let mut mem = [0u8; PAGE_SIZE as usize];
+		let mut heap = FreeingBumpHeapAllocator::new(1);
+
+		// when
+		// an item size of 16 must be used then
+		heap.allocate(&mut mem[..], 9).unwrap();
+
+		// then
+		assert_eq!(heap.total_size, PREFIX_SIZE + 16);
+	}
+
+	#[test]
+	fn should_calculate_total_heap_size_to_zero() {
+		// given
+		let mut mem = [0u8; PAGE_SIZE as usize];
+		let mut heap = FreeingBumpHeapAllocator::new(13);
+
+		// when
+		let ptr = heap.allocate(&mut mem[..], 42).unwrap();
+		assert_eq!(ptr, to_pointer(16 + PREFIX_SIZE));
+		heap.deallocate(&mut mem[..], ptr).unwrap();
+
+		// then
+		assert_eq!(heap.total_size, 0);
+	}
+
+	#[test]
+	fn should_calculate_total_size_of_zero() {
+		// given
+		let mut mem = [0u8; PAGE_SIZE as usize];
+		let mut heap = FreeingBumpHeapAllocator::new(19);
+
+		// when
+		for _ in 1..10 {
+			let ptr = heap.allocate(&mut mem[..], 42).unwrap();
+			heap.deallocate(&mut mem[..], ptr).unwrap();
+		}
+
+		// then
+		assert_eq!(heap.total_size, 0);
+	}
+
+	#[test]
+	fn should_read_and_write_u64_correctly() {
+		// given
+		let mut mem = [0u8; PAGE_SIZE as usize];
+		let heap = FreeingBumpHeapAllocator::new(16);
+
+		// when
+		heap.set_heap_u64(&mut mem[..], 40, 4480113).unwrap();
+
+		// then
+		let value = heap.get_heap_u64(&mut mem[..], 40).unwrap();
+		assert_eq!(value, 4480113);
+	}
+
+	#[test]
+	fn should_get_item_size_from_index() {
+		// given
+		let index = 0;
+
+		// when
+		let item_size = FreeingBumpHeapAllocator::get_item_size_from_index(index);
+
+		// then
+		assert_eq!(item_size, 8);
+	}
+
+	#[test]
+	fn should_get_max_item_size_from_index() {
+		// given
+		let index = 21;
+
+		// when
+		let item_size = FreeingBumpHeapAllocator::get_item_size_from_index(index);
+
+		// then
+		assert_eq!(item_size as u32, MAX_POSSIBLE_ALLOCATION);
+	}
+
+}

substrate/client/executor/src/error.rs

@@ -0,0 +1,140 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Rust executor possible errors.
+
+use serializer;
+use wasmi;
+#[cfg(feature = "wasmtime")]
+use wasmtime_jit::{ActionError, SetupError};
+
+/// Result type alias.
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Error type.
+#[derive(Debug, derive_more::Display, derive_more::From)]
+pub enum Error {
+	/// Unserializable Data
+	InvalidData(serializer::Error),
+	/// Trap occured during execution
+	Trap(wasmi::Trap),
+	/// Wasmi loading/instantiating error
+	Wasmi(wasmi::Error),
+	/// Wasmtime action error
+	#[cfg(feature = "wasmtime")]
+	Wasmtime(ActionError),
+	/// Error in the API. Parameter is an error message.
+	ApiError(String),
+	/// Method is not found
+	#[display(fmt="Method not found: '{}'", _0)]
+	MethodNotFound(String),
+	/// Code is invalid (expected single byte)
+	#[display(fmt="Invalid Code: {}", _0)]
+	InvalidCode(String),
+	/// Could not get runtime version.
+	#[display(fmt="On-chain runtime does not specify version")]
+	VersionInvalid,
+	/// Externalities have failed.
+	#[display(fmt="Externalities error")]
+	Externalities,
+	/// Invalid index.
+	#[display(fmt="Invalid index provided")]
+	InvalidIndex,
+	/// Invalid return type.
+	#[display(fmt="Invalid type returned (should be u64)")]
+	InvalidReturn,
+	/// Runtime failed.
+	#[display(fmt="Runtime error")]
+	Runtime,
+	/// Invalid memory reference.
+	#[display(fmt="Invalid memory reference")]
+	InvalidMemoryReference,
+	/// The runtime must provide a global named `__heap_base` of type i32 for specifying where the
+	/// allocator is allowed to place its data.
+	#[display(fmt="The runtime doesn't provide a global named `__heap_base`")]
+	HeapBaseNotFoundOrInvalid,
+	/// The runtime WebAssembly module is not allowed to have the `start` function.
+	#[display(fmt="The runtime has the `start` function")]
+	RuntimeHasStartFn,
+	/// Some other error occurred
+	Other(String),
+	/// Some error occurred in the allocator
+	#[display(fmt="Error in allocator: {}", _0)]
+	Allocator(&'static str),
+	/// The allocator ran out of space.
+	#[display(fmt="Allocator ran out of space")]
+	AllocatorOutOfSpace,
+	/// Someone tried to allocate more memory than the allowed maximum per allocation.
+	#[display(fmt="Requested allocation size is too large")]
+	RequestedAllocationTooLarge,
+	/// Execution of a host function failed.
+	#[display(fmt="Host function {} execution failed with: {}", _0, _1)]
+	FunctionExecution(String, String),
+}
+
+impl std::error::Error for Error {
+	fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+		match self {
+			Error::InvalidData(ref err) => Some(err),
+			Error::Trap(ref err) => Some(err),
+			Error::Wasmi(ref err) => Some(err),
+			_ => None,
+		}
+	}
+}
+
+impl wasmi::HostError for Error {}
+
+impl From<String> for Error {
+	fn from(err: String) -> Error {
+		Error::Other(err)
+	}
+}
+
+impl From<WasmError> for Error {
+	fn from(err: WasmError) -> Error {
+		Error::Other(err.to_string())
+	}
+}
+
+/// Type for errors occurring during Wasm runtime construction.
+#[derive(Debug, derive_more::Display)]
+pub enum WasmError {
+	/// Code could not be read from the state.
+	CodeNotFound,
+	/// Failure to reinitialize runtime instance from snapshot.
+	ApplySnapshotFailed,
+	/// Failure to erase the wasm memory.
+	///
+	/// Depending on the implementation might mean failure of allocating memory.
+	ErasingFailed(String),
+	/// Wasm code failed validation.
+	InvalidModule,
+	/// Wasm code could not be deserialized.
+	CantDeserializeWasm,
+	/// The module does not export a linear memory named `memory`.
+	InvalidMemory,
+	/// The number of heap pages requested is disallowed by the module.
+	InvalidHeapPages,
+	/// Instantiation error.
+	Instantiation(String),
+	/// The compiler does not support the host machine as a target.
+	#[cfg(feature = "wasmtime")]
+	MissingCompilerSupport(&'static str),
+	/// Wasmtime setup error.
+	#[cfg(feature = "wasmtime")]
+	WasmtimeSetup(SetupError),
+}

substrate/client/executor/src/integration_tests/mod.rs

@@ -0,0 +1,459 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+mod sandbox;
+
+use codec::{Encode, Decode};
+use hex_literal::hex;
+use primitives::{
+	Blake2Hasher, blake2_128, blake2_256, ed25519, sr25519, map, Pair, offchain::OffchainExt,
+	traits::Externalities,
+};
+use runtime_test::WASM_BINARY;
+use state_machine::TestExternalities as CoreTestExternalities;
+use substrate_offchain::testing;
+use test_case::test_case;
+use trie::{TrieConfiguration, trie_types::Layout};
+
+use crate::WasmExecutionMethod;
+
+pub type TestExternalities = CoreTestExternalities<Blake2Hasher, u64>;
+
+fn call_in_wasm<E: Externalities>(
+	function: &str,
+	call_data: &[u8],
+	execution_method: WasmExecutionMethod,
+	ext: &mut E,
+	code: &[u8],
+	heap_pages: u64,
+) -> crate::error::Result<Vec<u8>> {
+	crate::call_in_wasm::<E, runtime_io::SubstrateHostFunctions>(
+		function,
+		call_data,
+		execution_method,
+		ext,
+		code,
+		heap_pages,
+	)
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn returning_should_work(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+
+	let output = call_in_wasm(
+		"test_empty_return",
+		&[],
+		wasm_method,
+		&mut ext,
+		&test_code[..],
+		8,
+	).unwrap();
+	assert_eq!(output, vec![0u8; 0]);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn panicking_should_work(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+
+	let output = call_in_wasm(
+		"test_panic",
+		&[],
+		wasm_method,
+		&mut ext,
+		&test_code[..],
+		8,
+	);
+	assert!(output.is_err());
+
+	let output = call_in_wasm(
+		"test_conditional_panic",
+		&[0],
+		wasm_method,
+		&mut ext,
+		&test_code[..],
+		8,
+	);
+	assert_eq!(Decode::decode(&mut &output.unwrap()[..]), Ok(Vec::<u8>::new()));
+
+	let output = call_in_wasm(
+		"test_conditional_panic",
+		&vec![2].encode(),
+		wasm_method,
+		&mut ext,
+		&test_code[..],
+		8,
+	);
+	assert!(output.is_err());
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn storage_should_work(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+
+	{
+		let mut ext = ext.ext();
+		ext.set_storage(b"foo".to_vec(), b"bar".to_vec());
+		let test_code = WASM_BINARY;
+
+		let output = call_in_wasm(
+			"test_data_in",
+			&b"Hello world".to_vec().encode(),
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap();
+
+		assert_eq!(output, b"all ok!".to_vec().encode());
+	}
+
+	let expected = TestExternalities::new((map![
+			b"input".to_vec() => b"Hello world".to_vec(),
+			b"foo".to_vec() => b"bar".to_vec(),
+			b"baz".to_vec() => b"bar".to_vec()
+		], map![]));
+	assert_eq!(ext, expected);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn clear_prefix_should_work(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	{
+		let mut ext = ext.ext();
+		ext.set_storage(b"aaa".to_vec(), b"1".to_vec());
+		ext.set_storage(b"aab".to_vec(), b"2".to_vec());
+		ext.set_storage(b"aba".to_vec(), b"3".to_vec());
+		ext.set_storage(b"abb".to_vec(), b"4".to_vec());
+		ext.set_storage(b"bbb".to_vec(), b"5".to_vec());
+		let test_code = WASM_BINARY;
+
+		// This will clear all entries which prefix is "ab".
+		let output = call_in_wasm(
+			"test_clear_prefix",
+			&b"ab".to_vec().encode(),
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap();
+
+		assert_eq!(output, b"all ok!".to_vec().encode());
+	}
+
+	let expected = TestExternalities::new((map![
+			b"aaa".to_vec() => b"1".to_vec(),
+			b"aab".to_vec() => b"2".to_vec(),
+			b"bbb".to_vec() => b"5".to_vec()
+		], map![]));
+	assert_eq!(expected, ext);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn blake2_256_should_work(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+	assert_eq!(
+		call_in_wasm(
+			"test_blake2_256",
+			&[0],
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		blake2_256(&b""[..]).to_vec().encode(),
+	);
+	assert_eq!(
+		call_in_wasm(
+			"test_blake2_256",
+			&b"Hello world!".to_vec().encode(),
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		blake2_256(&b"Hello world!"[..]).to_vec().encode(),
+	);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn blake2_128_should_work(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+	assert_eq!(
+		call_in_wasm(
+			"test_blake2_128",
+			&[0],
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		blake2_128(&b""[..]).to_vec().encode(),
+	);
+	assert_eq!(
+		call_in_wasm(
+			"test_blake2_128",
+			&b"Hello world!".to_vec().encode(),
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		blake2_128(&b"Hello world!"[..]).to_vec().encode(),
+	);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn twox_256_should_work(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+	assert_eq!(
+		call_in_wasm(
+			"test_twox_256",
+			&[0],
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		hex!(
+				"99e9d85137db46ef4bbea33613baafd56f963c64b1f3685a4eb4abd67ff6203a"
+			).to_vec().encode(),
+	);
+	assert_eq!(
+		call_in_wasm(
+			"test_twox_256",
+			&b"Hello world!".to_vec().encode(),
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		hex!(
+				"b27dfd7f223f177f2a13647b533599af0c07f68bda23d96d059da2b451a35a74"
+			).to_vec().encode(),
+	);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn twox_128_should_work(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+	assert_eq!(
+		call_in_wasm(
+			"test_twox_128",
+			&[0],
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		hex!("99e9d85137db46ef4bbea33613baafd5").to_vec().encode(),
+	);
+	assert_eq!(
+		call_in_wasm(
+			"test_twox_128",
+			&b"Hello world!".to_vec().encode(),
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		hex!("b27dfd7f223f177f2a13647b533599af").to_vec().encode(),
+	);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn ed25519_verify_should_work(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+	let key = ed25519::Pair::from_seed(&blake2_256(b"test"));
+	let sig = key.sign(b"all ok!");
+	let mut calldata = vec![];
+	calldata.extend_from_slice(key.public().as_ref());
+	calldata.extend_from_slice(sig.as_ref());
+
+	assert_eq!(
+		call_in_wasm(
+			"test_ed25519_verify",
+			&calldata.encode(),
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		true.encode(),
+	);
+
+	let other_sig = key.sign(b"all is not ok!");
+	let mut calldata = vec![];
+	calldata.extend_from_slice(key.public().as_ref());
+	calldata.extend_from_slice(other_sig.as_ref());
+
+	assert_eq!(
+		call_in_wasm(
+			"test_ed25519_verify",
+			&calldata.encode(),
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		false.encode(),
+	);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn sr25519_verify_should_work(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+	let key = sr25519::Pair::from_seed(&blake2_256(b"test"));
+	let sig = key.sign(b"all ok!");
+	let mut calldata = vec![];
+	calldata.extend_from_slice(key.public().as_ref());
+	calldata.extend_from_slice(sig.as_ref());
+
+	assert_eq!(
+		call_in_wasm(
+			"test_sr25519_verify",
+			&calldata.encode(),
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		true.encode(),
+	);
+
+	let other_sig = key.sign(b"all is not ok!");
+	let mut calldata = vec![];
+	calldata.extend_from_slice(key.public().as_ref());
+	calldata.extend_from_slice(other_sig.as_ref());
+
+	assert_eq!(
+		call_in_wasm(
+			"test_sr25519_verify",
+			&calldata.encode(),
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		false.encode(),
+	);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn ordered_trie_root_should_work(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let trie_input = vec![b"zero".to_vec(), b"one".to_vec(), b"two".to_vec()];
+	let test_code = WASM_BINARY;
+	assert_eq!(
+		call_in_wasm(
+			"test_ordered_trie_root",
+			&[0],
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		Layout::<Blake2Hasher>::ordered_trie_root(trie_input.iter()).as_bytes().encode(),
+	);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn offchain_local_storage_should_work(wasm_method: WasmExecutionMethod) {
+	use client_api::OffchainStorage;
+
+	let mut ext = TestExternalities::default();
+	let (offchain, state) = testing::TestOffchainExt::new();
+	ext.register_extension(OffchainExt::new(offchain));
+	let test_code = WASM_BINARY;
+	let mut ext = ext.ext();
+	assert_eq!(
+		call_in_wasm(
+			"test_offchain_local_storage",
+			&[0],
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		true.encode(),
+	);
+	assert_eq!(state.read().persistent_storage.get(b"", b"test"), Some(vec![]));
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn offchain_http_should_work(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let (offchain, state) = testing::TestOffchainExt::new();
+	ext.register_extension(OffchainExt::new(offchain));
+	state.write().expect_request(
+		0,
+		testing::PendingRequest {
+			method: "POST".into(),
+			uri: "http://localhost:12345".into(),
+			body: vec![1, 2, 3, 4],
+			headers: vec![("X-Auth".to_owned(), "test".to_owned())],
+			sent: true,
+			response: Some(vec![1, 2, 3]),
+			response_headers: vec![("X-Auth".to_owned(), "hello".to_owned())],
+			..Default::default()
+		},
+	);
+
+	let test_code = WASM_BINARY;
+	let mut ext = ext.ext();
+	assert_eq!(
+		call_in_wasm(
+			"test_offchain_http",
+			&[0],
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		true.encode(),
+	);
+}
+

substrate/client/executor/src/integration_tests/sandbox.rs

@@ -0,0 +1,360 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use super::{TestExternalities, call_in_wasm};
+use crate::WasmExecutionMethod;
+
+use codec::Encode;
+use runtime_test::WASM_BINARY;
+use test_case::test_case;
+use wabt;
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn sandbox_should_work(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+
+	let code = wabt::wat2wasm(r#"
+		(module
+			(import "env" "assert" (func $assert (param i32)))
+			(import "env" "inc_counter" (func $inc_counter (param i32) (result i32)))
+			(func (export "call")
+				(drop
+					(call $inc_counter (i32.const 5))
+				)
+
+				(call $inc_counter (i32.const 3))
+				;; current counter value is on the stack
+
+				;; check whether current == 8
+				i32.const 8
+				i32.eq
+
+				call $assert
+			)
+		)
+		"#).unwrap().encode();
+
+	assert_eq!(
+		call_in_wasm(
+			"test_sandbox",
+			&code,
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		true.encode(),
+	);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn sandbox_trap(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+
+	let code = wabt::wat2wasm(r#"
+		(module
+			(import "env" "assert" (func $assert (param i32)))
+			(func (export "call")
+				i32.const 0
+				call $assert
+			)
+		)
+		"#).unwrap();
+
+	assert_eq!(
+		call_in_wasm(
+			"test_sandbox",
+			&code,
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		vec![0],
+	);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+#[should_panic(expected = "Allocator ran out of space")]
+fn sandbox_should_trap_when_heap_exhausted(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+
+	let code = wabt::wat2wasm(r#"
+		(module
+			(import "env" "assert" (func $assert (param i32)))
+			(func (export "call")
+				i32.const 0
+				call $assert
+			)
+		)
+		"#).unwrap().encode();
+
+	call_in_wasm(
+		"test_exhaust_heap",
+		&code,
+		wasm_method,
+		&mut ext,
+		&test_code[..],
+		8,
+	).unwrap();
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn start_called(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+
+	let code = wabt::wat2wasm(r#"
+		(module
+			(import "env" "assert" (func $assert (param i32)))
+			(import "env" "inc_counter" (func $inc_counter (param i32) (result i32)))
+
+			;; Start function
+			(start $start)
+			(func $start
+				;; Increment counter by 1
+				(drop
+					(call $inc_counter (i32.const 1))
+				)
+			)
+
+			(func (export "call")
+				;; Increment counter by 1. The current value is placed on the stack.
+				(call $inc_counter (i32.const 1))
+
+				;; Counter is incremented twice by 1, once there and once in `start` func.
+				;; So check the returned value is equal to 2.
+				i32.const 2
+				i32.eq
+				call $assert
+			)
+		)
+		"#).unwrap().encode();
+
+	assert_eq!(
+		call_in_wasm(
+			"test_sandbox",
+			&code,
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		true.encode(),
+	);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn invoke_args(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+
+	let code = wabt::wat2wasm(r#"
+		(module
+			(import "env" "assert" (func $assert (param i32)))
+
+			(func (export "call") (param $x i32) (param $y i64)
+				;; assert that $x = 0x12345678
+				(call $assert
+					(i32.eq
+						(get_local $x)
+						(i32.const 0x12345678)
+					)
+				)
+
+				(call $assert
+					(i64.eq
+						(get_local $y)
+						(i64.const 0x1234567887654321)
+					)
+				)
+			)
+		)
+		"#).unwrap().encode();
+
+	assert_eq!(
+		call_in_wasm(
+			"test_sandbox_args",
+			&code,
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		true.encode(),
+	);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn return_val(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+
+	let code = wabt::wat2wasm(r#"
+		(module
+			(func (export "call") (param $x i32) (result i32)
+				(i32.add
+					(get_local $x)
+					(i32.const 1)
+				)
+			)
+		)
+		"#).unwrap().encode();
+
+	assert_eq!(
+		call_in_wasm(
+			"test_sandbox_return_val",
+			&code,
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		true.encode(),
+	);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn unlinkable_module(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+
+	let code = wabt::wat2wasm(r#"
+		(module
+			(import "env" "non-existent" (func))
+
+			(func (export "call")
+			)
+		)
+		"#).unwrap().encode();
+
+	assert_eq!(
+		call_in_wasm(
+			"test_sandbox_instantiate",
+			&code,
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		1u8.encode(),
+	);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn corrupted_module(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+
+	// Corrupted wasm file
+	let code = vec![0u8, 0, 0, 0, 1, 0, 0, 0].encode();
+
+	assert_eq!(
+		call_in_wasm(
+			"test_sandbox_instantiate",
+			&code,
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		1u8.encode(),
+	);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn start_fn_ok(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+
+	let code = wabt::wat2wasm(r#"
+		(module
+			(func (export "call")
+			)
+
+			(func $start
+			)
+
+			(start $start)
+		)
+		"#).unwrap().encode();
+
+	assert_eq!(
+		call_in_wasm(
+			"test_sandbox_instantiate",
+			&code,
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		0u8.encode(),
+	);
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn start_fn_traps(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+	let test_code = WASM_BINARY;
+
+	let code = wabt::wat2wasm(r#"
+		(module
+			(func (export "call")
+			)
+
+			(func $start
+				unreachable
+			)
+
+			(start $start)
+		)
+		"#).unwrap().encode();
+
+	assert_eq!(
+		call_in_wasm(
+			"test_sandbox_instantiate",
+			&code,
+			wasm_method,
+			&mut ext,
+			&test_code[..],
+			8,
+		).unwrap(),
+		2u8.encode(),
+	);
+}

substrate/client/executor/src/lib.rs

@@ -0,0 +1,116 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! A crate that provides means of executing/dispatching calls into the runtime.
+//!
+//! There are a few responsibilities of this crate at the moment:
+//!
+//! - It provides an implementation of a common entrypoint for calling into the runtime, both
+//! wasm and compiled.
+//! - It defines the environment for the wasm execution, namely the host functions that are to be
+//! provided into the wasm runtime module.
+//! - It also provides the required infrastructure for executing the current wasm runtime (specified
+//! by the current value of `:code` in the provided externalities), i.e. interfacing with
+//! wasm engine used, instance cache.
+
+#![warn(missing_docs)]
+#![recursion_limit="128"]
+
+#[macro_use]
+mod wasm_utils;
+mod wasmi_execution;
+#[macro_use]
+mod native_executor;
+mod sandbox;
+mod allocator;
+pub mod deprecated_host_interface;
+mod wasm_runtime;
+#[cfg(feature = "wasmtime")]
+mod wasmtime;
+#[cfg(test)]
+mod integration_tests;
+
+pub mod error;
+pub use wasmi;
+pub use native_executor::{with_native_environment, NativeExecutor, NativeExecutionDispatch};
+pub use runtime_version::{RuntimeVersion, NativeVersion};
+pub use codec::Codec;
+#[doc(hidden)]
+pub use primitives::traits::Externalities;
+#[doc(hidden)]
+pub use wasm_interface;
+pub use wasm_runtime::WasmExecutionMethod;
+
+/// Call the given `function` in the given wasm `code`.
+///
+/// The signature of `function` needs to follow the default Substrate function signature.
+///
+/// - `call_data`: Will be given as input parameters to `function`
+/// - `execution_method`: The execution method to use.
+/// - `ext`: The externalities that should be set while executing the wasm function.
+/// - `heap_pages`: The number of heap pages to allocate.
+///
+/// Returns the `Vec<u8>` that contains the return value of the function.
+pub fn call_in_wasm<E: Externalities, HF: wasm_interface::HostFunctions>(
+	function: &str,
+	call_data: &[u8],
+	execution_method: WasmExecutionMethod,
+	ext: &mut E,
+	code: &[u8],
+	heap_pages: u64,
+) -> error::Result<Vec<u8>> {
+	let mut instance = wasm_runtime::create_wasm_runtime_with_code(
+		execution_method,
+		heap_pages,
+		code,
+		HF::host_functions(),
+	)?;
+	instance.call(ext, function, call_data)
+}
+
+/// Provides runtime information.
+pub trait RuntimeInfo {
+	/// Native runtime information.
+	fn native_version(&self) -> &NativeVersion;
+
+	/// Extract RuntimeVersion of given :code block
+	fn runtime_version<E: Externalities> (
+		&self,
+		ext: &mut E,
+	) -> Option<RuntimeVersion>;
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use runtime_test::WASM_BINARY;
+	use runtime_io::TestExternalities;
+
+	#[test]
+	fn call_in_interpreted_wasm_works() {
+		let mut ext = TestExternalities::default();
+		let mut ext = ext.ext();
+		let res = call_in_wasm::<_, runtime_io::SubstrateHostFunctions>(
+			"test_empty_return",
+			&[],
+			WasmExecutionMethod::Interpreted,
+			&mut ext,
+			&WASM_BINARY,
+			8,
+		).unwrap();
+		assert_eq!(res, vec![0u8; 0]);
+	}
+}

substrate/client/executor/src/native_executor.rs

@@ -0,0 +1,285 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{
+	RuntimeInfo, error::{Error, Result},
+	wasm_runtime::{RuntimesCache, WasmExecutionMethod, WasmRuntime},
+};
+
+use runtime_version::{NativeVersion, RuntimeVersion};
+
+use codec::{Decode, Encode};
+
+use primitives::{NativeOrEncoded, traits::{CodeExecutor, Externalities}};
+
+use log::{trace, warn};
+
+use std::{result, cell::RefCell, panic::{UnwindSafe, AssertUnwindSafe}};
+
+use wasm_interface::{HostFunctions, Function};
+
+thread_local! {
+	static RUNTIMES_CACHE: RefCell<RuntimesCache> = RefCell::new(RuntimesCache::new());
+}
+
+/// Default num of pages for the heap
+const DEFAULT_HEAP_PAGES: u64 = 1024;
+
+pub(crate) fn safe_call<F, U>(f: F) -> Result<U>
+	where F: UnwindSafe + FnOnce() -> U
+{
+	// Substrate uses custom panic hook that terminates process on panic. Disable termination for the native call.
+	let _guard = panic_handler::AbortGuard::force_unwind();
+	std::panic::catch_unwind(f).map_err(|_| Error::Runtime)
+}
+
+/// Set up the externalities and safe calling environment to execute calls to a native runtime.
+///
+/// If the inner closure panics, it will be caught and return an error.
+pub fn with_native_environment<F, U>(ext: &mut dyn Externalities, f: F) -> Result<U>
+	where F: UnwindSafe + FnOnce() -> U
+{
+	externalities::set_and_run_with_externalities(ext, move || safe_call(f))
+}
+
+/// Delegate for dispatching a CodeExecutor call.
+///
+/// By dispatching we mean that we execute a runtime function specified by it's name.
+pub trait NativeExecutionDispatch: Send + Sync {
+	/// Dispatch a method in the runtime.
+	///
+	/// If the method with the specified name doesn't exist then `Err` is returned.
+	fn dispatch(ext: &mut dyn Externalities, method: &str, data: &[u8]) -> Result<Vec<u8>>;
+
+	/// Provide native runtime version.
+	fn native_version() -> NativeVersion;
+}
+
+/// A generic `CodeExecutor` implementation that uses a delegate to determine wasm code equivalence
+/// and dispatch to native code when possible, falling back on `WasmExecutor` when not.
+pub struct NativeExecutor<D> {
+	/// Dummy field to avoid the compiler complaining about us not using `D`.
+	_dummy: std::marker::PhantomData<D>,
+	/// Method used to execute fallback Wasm code.
+	fallback_method: WasmExecutionMethod,
+	/// Native runtime version info.
+	native_version: NativeVersion,
+	/// The number of 64KB pages to allocate for Wasm execution.
+	default_heap_pages: u64,
+	/// The host functions registered with this instance.
+	host_functions: Vec<&'static dyn Function>,
+}
+
+impl<D: NativeExecutionDispatch> NativeExecutor<D> {
+	/// Create new instance.
+	///
+	/// # Parameters
+	///
+	/// `fallback_method` - Method used to execute fallback Wasm code.
+	///
+	/// `default_heap_pages` - Number of 64KB pages to allocate for Wasm execution.
+	/// 	Defaults to `DEFAULT_HEAP_PAGES` if `None` is provided.
+	pub fn new(fallback_method: WasmExecutionMethod, default_heap_pages: Option<u64>) -> Self {
+		let mut host_functions = runtime_io::SubstrateHostFunctions::host_functions();
+		// Add the old and deprecated host functions as well, so that we support old wasm runtimes.
+		host_functions.extend(
+			crate::deprecated_host_interface::SubstrateExternals::host_functions(),
+		);
+
+		NativeExecutor {
+			_dummy: Default::default(),
+			fallback_method,
+			native_version: D::native_version(),
+			default_heap_pages: default_heap_pages.unwrap_or(DEFAULT_HEAP_PAGES),
+			host_functions,
+		}
+	}
+
+	/// Execute the given closure `f` with the latest runtime (based on the `CODE` key in `ext`).
+	///
+	/// The closure `f` is expected to return `Err(_)` when there happened a `panic!` in native code
+	/// while executing the runtime in Wasm. If a `panic!` occurred, the runtime is invalidated to
+	/// prevent any poisoned state. Native runtime execution does not need to report back
+	/// any `panic!`.
+	///
+	/// # Safety
+	///
+	/// `runtime` and `ext` are given as `AssertUnwindSafe` to the closure. As described above, the
+	/// runtime is invalidated on any `panic!` to prevent a poisoned state. `ext` is already
+	/// implicitly handled as unwind safe, as we store it in a global variable while executing the
+	/// native runtime.
+	fn with_runtime<E, R>(
+		&self,
+		ext: &mut E,
+		f: impl for<'a> FnOnce(
+			AssertUnwindSafe<&'a mut (dyn WasmRuntime + 'static)>,
+			&'a RuntimeVersion,
+			AssertUnwindSafe<&'a mut E>,
+		) -> Result<Result<R>>,
+	) -> Result<R> where E: Externalities {
+		RUNTIMES_CACHE.with(|cache| {
+			let mut cache = cache.borrow_mut();
+			let (runtime, version, code_hash) = cache.fetch_runtime(
+				ext,
+				self.fallback_method,
+				self.default_heap_pages,
+				&self.host_functions,
+			)?;
+
+			let runtime = AssertUnwindSafe(runtime);
+			let ext = AssertUnwindSafe(ext);
+
+			match f(runtime, version, ext) {
+				Ok(res) => res,
+				Err(e) => {
+					cache.invalidate_runtime(self.fallback_method, code_hash);
+					Err(e)
+				}
+			}
+		})
+	}
+}
+
+impl<D: NativeExecutionDispatch> Clone for NativeExecutor<D> {
+	fn clone(&self) -> Self {
+		NativeExecutor {
+			_dummy: Default::default(),
+			fallback_method: self.fallback_method,
+			native_version: D::native_version(),
+			default_heap_pages: self.default_heap_pages,
+			host_functions: self.host_functions.clone(),
+		}
+	}
+}
+
+impl<D: NativeExecutionDispatch> RuntimeInfo for NativeExecutor<D> {
+	fn native_version(&self) -> &NativeVersion {
+		&self.native_version
+	}
+
+	fn runtime_version<E: Externalities>(
+		&self,
+		ext: &mut E,
+	) -> Option<RuntimeVersion> {
+		match self.with_runtime(ext, |_runtime, version, _ext| Ok(Ok(version.clone()))) {
+			Ok(version) => Some(version),
+			Err(e) => {
+				warn!(target: "executor", "Failed to fetch runtime: {:?}", e);
+				None
+			}
+		}
+	}
+}
+
+impl<D: NativeExecutionDispatch> CodeExecutor for NativeExecutor<D> {
+	type Error = Error;
+
+	fn call
+	<
+		E: Externalities,
+		R: Decode + Encode + PartialEq,
+		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
+	>(
+		&self,
+		ext: &mut E,
+		method: &str,
+		data: &[u8],
+		use_native: bool,
+		native_call: Option<NC>,
+	) -> (Result<NativeOrEncoded<R>>, bool){
+		let mut used_native = false;
+		let result = self.with_runtime(ext, |mut runtime, onchain_version, mut ext| {
+			match (
+				use_native,
+				onchain_version.can_call_with(&self.native_version.runtime_version),
+				native_call,
+			) {
+				(_, false, _) => {
+					trace!(
+						target: "executor",
+						"Request for native execution failed (native: {}, chain: {})",
+						self.native_version.runtime_version,
+						onchain_version,
+					);
+
+					safe_call(
+						move || runtime.call(&mut **ext, method, data).map(NativeOrEncoded::Encoded)
+					)
+				}
+				(false, _, _) => {
+					safe_call(
+						move || runtime.call(&mut **ext, method, data).map(NativeOrEncoded::Encoded)
+					)
+				},
+				(true, true, Some(call)) => {
+					trace!(
+						target: "executor",
+						"Request for native execution with native call succeeded (native: {}, chain: {}).",
+						self.native_version.runtime_version,
+						onchain_version,
+					);
+
+					used_native = true;
+					let res = with_native_environment(&mut **ext, move || (call)())
+						.and_then(|r| r
+							.map(NativeOrEncoded::Native)
+							.map_err(|s| Error::ApiError(s.to_string()))
+						);
+
+					Ok(res)
+				}
+				_ => {
+					trace!(
+						target: "executor",
+						"Request for native execution succeeded (native: {}, chain: {})",
+						self.native_version.runtime_version,
+						onchain_version
+					);
+
+					used_native = true;
+					Ok(D::dispatch(&mut **ext, method, data).map(NativeOrEncoded::Encoded))
+				}
+			}
+		});
+		(result, used_native)
+	}
+}
+
+/// Implements a `NativeExecutionDispatch` for provided parameters.
+#[macro_export]
+macro_rules! native_executor_instance {
+	( $pub:vis $name:ident, $dispatcher:path, $version:path $(,)?) => {
+		/// A unit struct which implements `NativeExecutionDispatch` feeding in the hard-coded runtime.
+		$pub struct $name;
+		$crate::native_executor_instance!(IMPL $name, $dispatcher, $version);
+	};
+	(IMPL $name:ident, $dispatcher:path, $version:path) => {
+		impl $crate::NativeExecutionDispatch for $name {
+			fn dispatch(
+				ext: &mut $crate::Externalities,
+				method: &str,
+				data: &[u8]
+			) -> $crate::error::Result<Vec<u8>> {
+				$crate::with_native_environment(ext, move || $dispatcher(method, data))?
+					.ok_or_else(|| $crate::error::Error::MethodNotFound(method.to_owned()))
+			}
+
+			fn native_version() -> $crate::NativeVersion {
+				$version()
+			}
+		}
+	}
+}

substrate/client/executor/src/sandbox.rs

@@ -0,0 +1,584 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+#![warn(missing_docs)]
+
+//! This module implements sandboxing support in the runtime.
+
+use crate::error::{Result, Error};
+use std::{collections::HashMap, rc::Rc};
+use codec::{Decode, Encode};
+use primitives::sandbox as sandbox_primitives;
+use wasmi::{
+	Externals, ImportResolver, MemoryInstance, MemoryRef, Module, ModuleInstance,
+	ModuleRef, RuntimeArgs, RuntimeValue, Trap, TrapKind, memory_units::Pages,
+};
+use wasm_interface::{Pointer, WordSize};
+
+/// Index of a function inside the supervisor.
+///
+/// This is a typically an index in the default table of the supervisor, however
+/// the exact meaning of this index is depends on the implementation of dispatch function.
+#[derive(Copy, Clone, Debug, PartialEq)]
+pub struct SupervisorFuncIndex(usize);
+
+impl From<SupervisorFuncIndex> for usize {
+	fn from(index: SupervisorFuncIndex) -> Self {
+		index.0
+	}
+}
+
+/// Index of a function within guest index space.
+///
+/// This index is supposed to be used with as index for `Externals`.
+#[derive(Copy, Clone, Debug, PartialEq)]
+struct GuestFuncIndex(usize);
+
+/// This struct holds a mapping from guest index space to supervisor.
+struct GuestToSupervisorFunctionMapping {
+	funcs: Vec<SupervisorFuncIndex>,
+}
+
+impl GuestToSupervisorFunctionMapping {
+	fn new() -> GuestToSupervisorFunctionMapping {
+		GuestToSupervisorFunctionMapping { funcs: Vec::new() }
+	}
+
+	fn define(&mut self, supervisor_func: SupervisorFuncIndex) -> GuestFuncIndex {
+		let idx = self.funcs.len();
+		self.funcs.push(supervisor_func);
+		GuestFuncIndex(idx)
+	}
+
+	fn func_by_guest_index(&self, guest_func_idx: GuestFuncIndex) -> Option<SupervisorFuncIndex> {
+		self.funcs.get(guest_func_idx.0).cloned()
+	}
+}
+
+struct Imports {
+	func_map: HashMap<(Vec<u8>, Vec<u8>), GuestFuncIndex>,
+	memories_map: HashMap<(Vec<u8>, Vec<u8>), MemoryRef>,
+}
+
+impl ImportResolver for Imports {
+	fn resolve_func(
+		&self,
+		module_name: &str,
+		field_name: &str,
+		signature: &::wasmi::Signature,
+	) -> std::result::Result<wasmi::FuncRef, wasmi::Error> {
+		let key = (
+			module_name.as_bytes().to_owned(),
+			field_name.as_bytes().to_owned(),
+		);
+		let idx = *self.func_map.get(&key).ok_or_else(|| {
+			wasmi::Error::Instantiation(format!(
+				"Export {}:{} not found",
+				module_name, field_name
+			))
+		})?;
+		Ok(wasmi::FuncInstance::alloc_host(signature.clone(), idx.0))
+	}
+
+	fn resolve_memory(
+		&self,
+		module_name: &str,
+		field_name: &str,
+		_memory_type: &::wasmi::MemoryDescriptor,
+	) -> std::result::Result<MemoryRef, wasmi::Error> {
+		let key = (
+			module_name.as_bytes().to_vec(),
+			field_name.as_bytes().to_vec(),
+		);
+		let mem = self.memories_map
+			.get(&key)
+			.ok_or_else(|| {
+				wasmi::Error::Instantiation(format!(
+					"Export {}:{} not found",
+					module_name, field_name
+				))
+			})?
+			.clone();
+		Ok(mem)
+	}
+
+	fn resolve_global(
+		&self,
+		module_name: &str,
+		field_name: &str,
+		_global_type: &::wasmi::GlobalDescriptor,
+	) -> std::result::Result<wasmi::GlobalRef, wasmi::Error> {
+		Err(wasmi::Error::Instantiation(format!(
+			"Export {}:{} not found",
+			module_name, field_name
+		)))
+	}
+
+	fn resolve_table(
+		&self,
+		module_name: &str,
+		field_name: &str,
+		_table_type: &::wasmi::TableDescriptor,
+	) -> std::result::Result<wasmi::TableRef, wasmi::Error> {
+		Err(wasmi::Error::Instantiation(format!(
+			"Export {}:{} not found",
+			module_name, field_name
+		)))
+	}
+}
+
+/// This trait encapsulates sandboxing capabilities.
+///
+/// Note that this functions are only called in the `supervisor` context.
+pub trait SandboxCapabilities {
+	/// Represents a function reference into the supervisor environment.
+	type SupervisorFuncRef;
+
+	/// Returns a reference to an associated sandbox `Store`.
+	fn store(&self) -> &Store<Self::SupervisorFuncRef>;
+
+	/// Returns a mutable reference to an associated sandbox `Store`.
+	fn store_mut(&mut self) -> &mut Store<Self::SupervisorFuncRef>;
+
+	/// Allocate space of the specified length in the supervisor memory.
+	///
+	/// # Errors
+	///
+	/// Returns `Err` if allocation not possible or errors during heap management.
+	///
+	/// Returns pointer to the allocated block.
+	fn allocate(&mut self, len: WordSize) -> Result<Pointer<u8>>;
+
+	/// Deallocate space specified by the pointer that was previously returned by [`allocate`].
+	///
+	/// # Errors
+	///
+	/// Returns `Err` if deallocation not possible or because of errors in heap management.
+	///
+	/// [`allocate`]: #tymethod.allocate
+	fn deallocate(&mut self, ptr: Pointer<u8>) -> Result<()>;
+
+	/// Write `data` into the supervisor memory at offset specified by `ptr`.
+	///
+	/// # Errors
+	///
+	/// Returns `Err` if `ptr + data.len()` is out of bounds.
+	fn write_memory(&mut self, ptr: Pointer<u8>, data: &[u8]) -> Result<()>;
+
+	/// Read `len` bytes from the supervisor memory.
+	///
+	/// # Errors
+	///
+	/// Returns `Err` if `ptr + len` is out of bounds.
+	fn read_memory(&self, ptr: Pointer<u8>, len: WordSize) -> Result<Vec<u8>>;
+
+	/// Invoke a function in the supervisor environment.
+	///
+	/// This first invokes the dispatch_thunk function, passing in the function index of the
+	/// desired function to call and serialized arguments. The thunk calls the desired function
+	/// with the deserialized arguments, then serializes the result into memory and returns
+	/// reference. The pointer to and length of the result in linear memory is encoded into an i64,
+	/// with the upper 32 bits representing the pointer and the lower 32 bits representing the
+	/// length.
+	///
+	/// # Errors
+	///
+	/// Returns `Err` if the dispatch_thunk function has an incorrect signature or traps during
+	/// execution.
+	fn invoke(
+		&mut self,
+		dispatch_thunk: &Self::SupervisorFuncRef,
+		invoke_args_ptr: Pointer<u8>,
+		invoke_args_len: WordSize,
+		state: u32,
+		func_idx: SupervisorFuncIndex,
+	) -> Result<i64>;
+}
+
+/// Implementation of [`Externals`] that allows execution of guest module with
+/// [externals][`Externals`] that might refer functions defined by supervisor.
+///
+/// [`Externals`]: ../wasmi/trait.Externals.html
+pub struct GuestExternals<'a, FE: SandboxCapabilities + 'a> {
+	supervisor_externals: &'a mut FE,
+	sandbox_instance: &'a SandboxInstance<FE::SupervisorFuncRef>,
+	state: u32,
+}
+
+fn trap(msg: &'static str) -> Trap {
+	TrapKind::Host(Box::new(Error::Other(msg.into()))).into()
+}
+
+fn deserialize_result(serialized_result: &[u8]) -> std::result::Result<Option<RuntimeValue>, Trap> {
+	use self::sandbox_primitives::{HostError, ReturnValue};
+	let result_val = std::result::Result::<ReturnValue, HostError>::decode(&mut &serialized_result[..])
+		.map_err(|_| trap("Decoding Result<ReturnValue, HostError> failed!"))?;
+
+	match result_val {
+		Ok(return_value) => Ok(match return_value {
+			ReturnValue::Unit => None,
+			ReturnValue::Value(typed_value) => Some(RuntimeValue::from(typed_value)),
+		}),
+		Err(HostError) => Err(trap("Supervisor function returned sandbox::HostError")),
+	}
+}
+
+impl<'a, FE: SandboxCapabilities + 'a> Externals for GuestExternals<'a, FE> {
+	fn invoke_index(
+		&mut self,
+		index: usize,
+		args: RuntimeArgs,
+	) -> std::result::Result<Option<RuntimeValue>, Trap> {
+		// Make `index` typesafe again.
+		let index = GuestFuncIndex(index);
+
+		let func_idx = self.sandbox_instance
+			.guest_to_supervisor_mapping
+			.func_by_guest_index(index)
+			.expect(
+				"`invoke_index` is called with indexes registered via `FuncInstance::alloc_host`;
+					`FuncInstance::alloc_host` is called with indexes that was obtained from `guest_to_supervisor_mapping`;
+					`func_by_guest_index` called with `index` can't return `None`;
+					qed"
+			);
+
+		// Serialize arguments into a byte vector.
+		let invoke_args_data: Vec<u8> = args.as_ref()
+			.iter()
+			.cloned()
+			.map(sandbox_primitives::TypedValue::from)
+			.collect::<Vec<_>>()
+			.encode();
+
+		let state = self.state;
+
+		// Move serialized arguments inside the memory and invoke dispatch thunk and
+		// then free allocated memory.
+		let invoke_args_len = invoke_args_data.len() as WordSize;
+		let invoke_args_ptr = self.supervisor_externals.allocate(invoke_args_len)?;
+		self.supervisor_externals.write_memory(invoke_args_ptr, &invoke_args_data)?;
+		let result = self.supervisor_externals.invoke(
+			&self.sandbox_instance.dispatch_thunk,
+			invoke_args_ptr,
+			invoke_args_len,
+			state,
+			func_idx,
+		)?;
+		self.supervisor_externals.deallocate(invoke_args_ptr)?;
+
+		// dispatch_thunk returns pointer to serialized arguments.
+		// Unpack pointer and len of the serialized result data.
+		let (serialized_result_val_ptr, serialized_result_val_len) = {
+			// Cast to u64 to use zero-extension.
+			let v = result as u64;
+			let ptr = (v as u64 >> 32) as u32;
+			let len = (v & 0xFFFFFFFF) as u32;
+			(Pointer::new(ptr), len)
+		};
+
+		let serialized_result_val = self.supervisor_externals
+			.read_memory(serialized_result_val_ptr, serialized_result_val_len)?;
+		self.supervisor_externals
+			.deallocate(serialized_result_val_ptr)?;
+
+		deserialize_result(&serialized_result_val)
+	}
+}
+
+fn with_guest_externals<FE, R, F>(
+	supervisor_externals: &mut FE,
+	sandbox_instance: &SandboxInstance<FE::SupervisorFuncRef>,
+	state: u32,
+	f: F,
+) -> R
+where
+	FE: SandboxCapabilities,
+	F: FnOnce(&mut GuestExternals<FE>) -> R,
+{
+	let mut guest_externals = GuestExternals {
+		supervisor_externals,
+		sandbox_instance,
+		state,
+	};
+	f(&mut guest_externals)
+}
+
+/// Sandboxed instance of a wasm module.
+///
+/// It's primary purpose is to [`invoke`] exported functions on it.
+///
+/// All imports of this instance are specified at the creation time and
+/// imports are implemented by the supervisor.
+///
+/// Hence, in order to invoke an exported function on a sandboxed module instance,
+/// it's required to provide supervisor externals: it will be used to execute
+/// code in the supervisor context.
+///
+/// This is generic over a supervisor function reference type.
+///
+/// [`invoke`]: #method.invoke
+pub struct SandboxInstance<FR> {
+	instance: ModuleRef,
+	dispatch_thunk: FR,
+	guest_to_supervisor_mapping: GuestToSupervisorFunctionMapping,
+}
+
+impl<FR> SandboxInstance<FR> {
+	/// Invoke an exported function by a name.
+	///
+	/// `supervisor_externals` is required to execute the implementations
+	/// of the syscalls that published to a sandboxed module instance.
+	///
+	/// The `state` parameter can be used to provide custom data for
+	/// these syscall implementations.
+	pub fn invoke<FE: SandboxCapabilities<SupervisorFuncRef=FR>>(
+		&self,
+		export_name: &str,
+		args: &[RuntimeValue],
+		supervisor_externals: &mut FE,
+		state: u32,
+	) -> std::result::Result<Option<wasmi::RuntimeValue>, wasmi::Error> {
+		with_guest_externals(
+			supervisor_externals,
+			self,
+			state,
+			|guest_externals| {
+				self.instance
+					.invoke_export(export_name, args, guest_externals)
+			},
+		)
+	}
+}
+
+/// Error occurred during instantiation of a sandboxed module.
+pub enum InstantiationError {
+	/// Something wrong with the environment definition. It either can't
+	/// be decoded, have a reference to a non-existent or torn down memory instance.
+	EnvironmentDefinitionCorrupted,
+	/// Provided module isn't recognized as a valid webassembly binary.
+	ModuleDecoding,
+	/// Module is a well-formed webassembly binary but could not be instantiated. This could
+	/// happen because, e.g. the module imports entries not provided by the environment.
+	Instantiation,
+	/// Module is well-formed, instantiated and linked, but while executing the start function
+	/// a trap was generated.
+	StartTrapped,
+}
+
+fn decode_environment_definition(
+	raw_env_def: &[u8],
+	memories: &[Option<MemoryRef>],
+) -> std::result::Result<(Imports, GuestToSupervisorFunctionMapping), InstantiationError> {
+	let env_def = sandbox_primitives::EnvironmentDefinition::decode(&mut &raw_env_def[..])
+		.map_err(|_| InstantiationError::EnvironmentDefinitionCorrupted)?;
+
+	let mut func_map = HashMap::new();
+	let mut memories_map = HashMap::new();
+	let mut guest_to_supervisor_mapping = GuestToSupervisorFunctionMapping::new();
+
+	for entry in &env_def.entries {
+		let module = entry.module_name.clone();
+		let field = entry.field_name.clone();
+
+		match entry.entity {
+			sandbox_primitives::ExternEntity::Function(func_idx) => {
+				let externals_idx =
+					guest_to_supervisor_mapping.define(SupervisorFuncIndex(func_idx as usize));
+				func_map.insert((module, field), externals_idx);
+			}
+			sandbox_primitives::ExternEntity::Memory(memory_idx) => {
+				let memory_ref = memories
+					.get(memory_idx as usize)
+					.cloned()
+					.ok_or_else(|| InstantiationError::EnvironmentDefinitionCorrupted)?
+					.ok_or_else(|| InstantiationError::EnvironmentDefinitionCorrupted)?;
+				memories_map.insert((module, field), memory_ref);
+			}
+		}
+	}
+
+	Ok((
+		Imports {
+			func_map,
+			memories_map,
+		},
+		guest_to_supervisor_mapping,
+	))
+}
+
+/// Instantiate a guest module and return it's index in the store.
+///
+/// The guest module's code is specified in `wasm`. Environment that will be available to
+/// guest module is specified in `raw_env_def` (serialized version of [`EnvironmentDefinition`]).
+/// `dispatch_thunk` is used as function that handle calls from guests.
+///
+/// # Errors
+///
+/// Returns `Err` if any of the following conditions happens:
+///
+/// - `raw_env_def` can't be deserialized as a [`EnvironmentDefinition`].
+/// - Module in `wasm` is invalid or couldn't be instantiated.
+///
+/// [`EnvironmentDefinition`]: ../sandbox/struct.EnvironmentDefinition.html
+pub fn instantiate<FE: SandboxCapabilities>(
+	supervisor_externals: &mut FE,
+	dispatch_thunk: FE::SupervisorFuncRef,
+	wasm: &[u8],
+	raw_env_def: &[u8],
+	state: u32,
+) -> std::result::Result<u32, InstantiationError> {
+	let (imports, guest_to_supervisor_mapping) =
+		decode_environment_definition(raw_env_def, &supervisor_externals.store().memories)?;
+
+	let module = Module::from_buffer(wasm).map_err(|_| InstantiationError::ModuleDecoding)?;
+	let instance = ModuleInstance::new(&module, &imports).map_err(|_| InstantiationError::Instantiation)?;
+
+	let sandbox_instance = Rc::new(SandboxInstance {
+		// In general, it's not a very good idea to use `.not_started_instance()` for anything
+		// but for extracting memory and tables. But in this particular case, we are extracting
+		// for the purpose of running `start` function which should be ok.
+		instance: instance.not_started_instance().clone(),
+		dispatch_thunk,
+		guest_to_supervisor_mapping,
+	});
+
+	with_guest_externals(
+		supervisor_externals,
+		&sandbox_instance,
+		state,
+		|guest_externals| {
+			instance
+				.run_start(guest_externals)
+				.map_err(|_| InstantiationError::StartTrapped)
+		},
+	)?;
+
+	// At last, register the instance.
+	let instance_idx = supervisor_externals
+		.store_mut()
+		.register_sandbox_instance(sandbox_instance);
+	Ok(instance_idx)
+}
+
+/// This struct keeps track of all sandboxed components.
+///
+/// This is generic over a supervisor function reference type.
+pub struct Store<FR> {
+	// Memories and instances are `Some` untill torndown.
+	instances: Vec<Option<Rc<SandboxInstance<FR>>>>,
+	memories: Vec<Option<MemoryRef>>,
+}
+
+impl<FR> Store<FR> {
+	/// Create a new empty sandbox store.
+	pub fn new() -> Self {
+		Store {
+			instances: Vec::new(),
+			memories: Vec::new(),
+		}
+	}
+
+	/// Create a new memory instance and return it's index.
+	///
+	/// # Errors
+	///
+	/// Returns `Err` if the memory couldn't be created.
+	/// Typically happens if `initial` is more than `maximum`.
+	pub fn new_memory(&mut self, initial: u32, maximum: u32) -> Result<u32> {
+		let maximum = match maximum {
+			sandbox_primitives::MEM_UNLIMITED => None,
+			specified_limit => Some(Pages(specified_limit as usize)),
+		};
+
+		let mem =
+			MemoryInstance::alloc(
+				Pages(initial as usize),
+				maximum,
+			)?;
+
+		let mem_idx = self.memories.len();
+		self.memories.push(Some(mem));
+		Ok(mem_idx as u32)
+	}
+
+	/// Returns `SandboxInstance` by `instance_idx`.
+	///
+	/// # Errors
+	///
+	/// Returns `Err` If `instance_idx` isn't a valid index of an instance or
+	/// instance is already torndown.
+	pub fn instance(&self, instance_idx: u32) -> Result<Rc<SandboxInstance<FR>>> {
+		self.instances
+			.get(instance_idx as usize)
+			.cloned()
+			.ok_or_else(|| "Trying to access a non-existent instance")?
+			.ok_or_else(|| "Trying to access a torndown instance".into())
+	}
+
+	/// Returns reference to a memory instance by `memory_idx`.
+	///
+	/// # Errors
+	///
+	/// Returns `Err` If `memory_idx` isn't a valid index of an memory or
+	/// if memory has been torn down.
+	pub fn memory(&self, memory_idx: u32) -> Result<MemoryRef> {
+		self.memories
+			.get(memory_idx as usize)
+			.cloned()
+			.ok_or_else(|| "Trying to access a non-existent sandboxed memory")?
+			.ok_or_else(|| "Trying to access a torndown sandboxed memory".into())
+	}
+
+	/// Tear down the memory at the specified index.
+	///
+	/// # Errors
+	///
+	/// Returns `Err` if `memory_idx` isn't a valid index of an memory or
+	/// if it has been torn down.
+	pub fn memory_teardown(&mut self, memory_idx: u32) -> Result<()> {
+		match self.memories.get_mut(memory_idx as usize) {
+			None => Err("Trying to teardown a non-existent sandboxed memory".into()),
+			Some(None) => Err("Double teardown of a sandboxed memory".into()),
+			Some(memory) => {
+				*memory = None;
+				Ok(())
+			}
+		}
+	}
+
+	/// Tear down the instance at the specified index.
+	///
+	/// # Errors
+	///
+	/// Returns `Err` if `instance_idx` isn't a valid index of an instance or
+	/// if it has been torn down.
+	pub fn instance_teardown(&mut self, instance_idx: u32) -> Result<()> {
+		match self.instances.get_mut(instance_idx as usize) {
+			None => Err("Trying to teardown a non-existent instance".into()),
+			Some(None) => Err("Double teardown of an instance".into()),
+			Some(instance) => {
+				*instance = None;
+				Ok(())
+			}
+		}
+	}
+
+	fn register_sandbox_instance(&mut self, sandbox_instance: Rc<SandboxInstance<FR>>) -> u32 {
+		let instance_idx = self.instances.len();
+		self.instances.push(Some(sandbox_instance));
+		instance_idx as u32
+	}
+}

substrate/client/executor/src/wasm_runtime.rs

@@ -0,0 +1,271 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Traits and accessor functions for calling into the Substrate Wasm runtime.
+//!
+//! The primary means of accessing the runtimes is through a cache which saves the reusable
+//! components of the runtime that are expensive to initialize.
+
+use crate::{wasmi_execution, error::{Error, WasmError}};
+#[cfg(feature = "wasmtime")]
+use crate::wasmtime;
+use log::{trace, warn};
+
+use codec::Decode;
+
+use primitives::{storage::well_known_keys, traits::Externalities, H256};
+
+use runtime_version::RuntimeVersion;
+use std::{collections::hash_map::{Entry, HashMap}, panic::AssertUnwindSafe};
+
+use wasm_interface::Function;
+
+/// The Substrate Wasm runtime.
+pub trait WasmRuntime {
+	/// Attempt to update the number of heap pages available during execution.
+	///
+	/// Returns false if the update cannot be applied. The function is guaranteed to return true if
+	/// the heap pages would not change from its current value.
+	fn update_heap_pages(&mut self, heap_pages: u64) -> bool;
+
+	/// Return the host functions that are registered for this Wasm runtime.
+	fn host_functions(&self) -> &[&'static dyn Function];
+
+	/// Call a method in the Substrate runtime by name. Returns the encoded result on success.
+	fn call(&mut self, ext: &mut dyn Externalities, method: &str, data: &[u8])
+		-> Result<Vec<u8>, Error>;
+}
+
+/// Specification of different methods of executing the runtime Wasm code.
+#[derive(Debug, PartialEq, Eq, Hash, Copy, Clone)]
+pub enum WasmExecutionMethod {
+	/// Uses the Wasmi interpreter.
+	Interpreted,
+	/// Uses the Wasmtime compiled runtime.
+	#[cfg(feature = "wasmtime")]
+	Compiled,
+}
+
+/// A Wasm runtime object along with its cached runtime version.
+struct VersionedRuntime {
+	runtime: Box<dyn WasmRuntime>,
+	/// Runtime version according to `Core_version`.
+	version: RuntimeVersion,
+}
+
+/// Cache for the runtimes.
+///
+/// When an instance is requested for the first time it is added to this cache. Metadata is kept
+/// with the instance so that it can be efficiently reinitialized.
+///
+/// When using the Wasmi interpreter execution method, the metadata includes the initial memory and
+/// values of mutable globals. Follow-up requests to fetch a runtime return this one instance with
+/// the memory reset to the initial memory. So, one runtime instance is reused for every fetch
+/// request.
+///
+/// For now the cache grows indefinitely, but that should be fine for now since runtimes can only be
+/// upgraded rarely and there are no other ways to make the node to execute some other runtime.
+pub struct RuntimesCache {
+	/// A cache of runtime instances along with metadata, ready to be reused.
+	///
+	/// Instances are keyed by the Wasm execution method and the hash of their code.
+	instances: HashMap<(WasmExecutionMethod, [u8; 32]), Result<VersionedRuntime, WasmError>>,
+}
+
+impl RuntimesCache {
+	/// Creates a new instance of a runtimes cache.
+	pub fn new() -> RuntimesCache {
+		RuntimesCache {
+			instances: HashMap::new(),
+		}
+	}
+
+	/// Fetches an instance of the runtime.
+	///
+	/// On first use we create a new runtime instance, save it to the cache
+	/// and persist its initial memory.
+	///
+	/// Each subsequent request will return this instance, with its memory restored
+	/// to the persisted initial memory. Thus, we reuse one single runtime instance
+	/// for every `fetch_runtime` invocation.
+	///
+	/// # Parameters
+	///
+	/// `ext` - Externalities to use for the runtime. This is used for setting
+	/// up an initial runtime instance.
+	///
+	/// `default_heap_pages` - Number of 64KB pages to allocate for Wasm execution.
+	///
+	/// `host_functions` - The host functions that should be registered for the Wasm runtime.
+	///
+	/// # Return value
+	///
+	/// If no error occurred a tuple `(&mut WasmRuntime, H256)` is
+	/// returned. `H256` is the hash of the runtime code.
+	///
+	/// In case of failure one of two errors can be returned:
+	///
+	/// `Err::InvalidCode` is returned for runtime code issues.
+	///
+	/// `Error::InvalidMemoryReference` is returned if no memory export with the
+	/// identifier `memory` can be found in the runtime.
+	pub fn fetch_runtime<E: Externalities>(
+		&mut self,
+		ext: &mut E,
+		wasm_method: WasmExecutionMethod,
+		default_heap_pages: u64,
+		host_functions: &[&'static dyn Function],
+	) -> Result<(&mut (dyn WasmRuntime + 'static), &RuntimeVersion, H256), Error> {
+		let code_hash = ext
+			.original_storage_hash(well_known_keys::CODE)
+			.ok_or(Error::InvalidCode("`CODE` not found in storage.".into()))?;
+
+		let heap_pages = ext
+			.storage(well_known_keys::HEAP_PAGES)
+			.and_then(|pages| u64::decode(&mut &pages[..]).ok())
+			.unwrap_or(default_heap_pages);
+
+		let result = match self.instances.entry((wasm_method, code_hash.into())) {
+			Entry::Occupied(o) => {
+				let result = o.into_mut();
+				if let Ok(ref mut cached_runtime) = result {
+					let heap_pages_changed = !cached_runtime.runtime.update_heap_pages(heap_pages);
+					let host_functions_changed = cached_runtime.runtime.host_functions()
+						!= host_functions;
+					if heap_pages_changed || host_functions_changed {
+						let changed = if heap_pages_changed {
+							"heap_pages"
+						} else {
+							"host functions"
+						};
+
+						trace!(
+							target: "runtimes_cache",
+							"{} were changed. Reinstantiating the instance",
+							changed,
+						);
+						*result = create_versioned_wasm_runtime(
+							ext,
+							wasm_method,
+							heap_pages,
+							host_functions.into(),
+						);
+						if let Err(ref err) = result {
+							warn!(target: "runtimes_cache", "cannot create a runtime: {:?}", err);
+						}
+					}
+				}
+				result
+			},
+			Entry::Vacant(v) => {
+				trace!(target: "runtimes_cache", "no instance found in cache, creating now.");
+				let result = create_versioned_wasm_runtime(
+					ext,
+					wasm_method,
+					heap_pages,
+					host_functions.into(),
+				);
+				if let Err(ref err) = result {
+					warn!(target: "runtimes_cache", "cannot create a runtime: {:?}", err);
+				}
+				v.insert(result)
+			}
+		};
+
+		result.as_mut()
+			.map(|entry| (entry.runtime.as_mut(), &entry.version, code_hash))
+			.map_err(|ref e| Error::InvalidCode(format!("{:?}", e)))
+	}
+
+	/// Invalidate the runtime for the given `wasm_method` and `code_hash`.
+	///
+	/// Invalidation of a runtime is useful when there was a `panic!` in native while executing it.
+	/// The `panic!` maybe have brought the runtime into a poisoned state and so, it is better to
+	/// invalidate this runtime instance.
+	pub fn invalidate_runtime(
+		&mut self,
+		wasm_method: WasmExecutionMethod,
+		code_hash: H256,
+	) {
+		// Just remove the instance, it will be re-created the next time it is requested.
+		self.instances.remove(&(wasm_method, code_hash.into()));
+	}
+}
+
+/// Create a wasm runtime with the given `code`.
+pub fn create_wasm_runtime_with_code(
+	wasm_method: WasmExecutionMethod,
+	heap_pages: u64,
+	code: &[u8],
+	host_functions: Vec<&'static dyn Function>,
+) -> Result<Box<dyn WasmRuntime>, WasmError> {
+	match wasm_method {
+		WasmExecutionMethod::Interpreted =>
+			wasmi_execution::create_instance(code, heap_pages, host_functions)
+				.map(|runtime| -> Box<dyn WasmRuntime> { Box::new(runtime) }),
+		#[cfg(feature = "wasmtime")]
+		WasmExecutionMethod::Compiled =>
+			wasmtime::create_instance(code, heap_pages, host_functions)
+				.map(|runtime| -> Box<dyn WasmRuntime> { Box::new(runtime) }),
+	}
+}
+
+fn create_versioned_wasm_runtime<E: Externalities>(
+	ext: &mut E,
+	wasm_method: WasmExecutionMethod,
+	heap_pages: u64,
+	host_functions: Vec<&'static dyn Function>,
+) -> Result<VersionedRuntime, WasmError> {
+	let code = ext
+		.original_storage(well_known_keys::CODE)
+		.ok_or(WasmError::CodeNotFound)?;
+	let mut runtime = create_wasm_runtime_with_code(wasm_method, heap_pages, &code, host_functions)?;
+
+	// Call to determine runtime version.
+	let version_result = {
+		// `ext` is already implicitly handled as unwind safe, as we store it in a global variable.
+		let mut ext = AssertUnwindSafe(ext);
+
+		// The following unwind safety assertion is OK because if the method call panics, the
+		// runtime will be dropped.
+		let mut runtime = AssertUnwindSafe(runtime.as_mut());
+		crate::native_executor::safe_call(
+			move || runtime.call(&mut **ext, "Core_version", &[])
+		).map_err(|_| WasmError::Instantiation("panic in call to get runtime version".into()))?
+	};
+	let encoded_version = version_result
+		.map_err(|e| WasmError::Instantiation(format!("failed to call \"Core_version\": {}", e)))?;
+	let version = RuntimeVersion::decode(&mut encoded_version.as_slice())
+		.map_err(|_| WasmError::Instantiation("failed to decode \"Core_version\" result".into()))?;
+
+	Ok(VersionedRuntime {
+		runtime,
+		version,
+	})
+}
+
+#[cfg(test)]
+mod tests {
+	use wasm_interface::HostFunctions;
+
+	#[test]
+	fn host_functions_are_equal() {
+		let host_functions = runtime_io::SubstrateHostFunctions::host_functions();
+
+		let equal = &host_functions[..] == &host_functions[..];
+		assert!(equal, "Host functions are not equal");
+	}
+}

substrate/client/executor/src/wasm_utils.rs

@@ -0,0 +1,186 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Utilities for defining the wasm host environment.
+
+use wasm_interface::{Pointer, WordSize};
+
+/// Converts arguments into respective WASM types.
+#[macro_export]
+macro_rules! convert_args {
+	() => ([]);
+	( $( $t:ty ),* ) => ( [ $( <$t as $crate::wasm_interface::IntoValue>::VALUE_TYPE, )* ] );
+}
+
+/// Generates a WASM signature for given list of parameters.
+#[macro_export]
+macro_rules! gen_signature {
+	( ( $( $params: ty ),* ) ) => (
+		$crate::wasm_interface::Signature {
+			args: std::borrow::Cow::Borrowed(&convert_args!( $( $params ),* )[..]),
+			return_value: None,
+		}
+	);
+	( ( $( $params: ty ),* ) -> $returns:ty ) => (
+		$crate::wasm_interface::Signature {
+			args: std::borrow::Cow::Borrowed(&convert_args!( $( $params ),* )[..]),
+			return_value: Some(<$returns as $crate::wasm_interface::IntoValue>::VALUE_TYPE),
+		}
+	);
+}
+
+macro_rules! gen_functions {
+	(@INTERNAL
+		{ $( $generated:tt )* }
+		$context:ident,
+	) => (
+		vec![ $( $generated )* ]
+	);
+	(@INTERNAL
+		{ $( $generated:tt )* }
+		$context:ident,
+		$name:ident ( $( $names:ident: $params:ty ),* ) $( -> $returns:ty )? { $( $body:tt )* }
+		$( $tail:tt )*
+	) => (
+		gen_functions! {
+			@INTERNAL
+			{
+				$( $generated )*
+				{
+					struct $name;
+
+					#[allow(unused)]
+					impl $crate::wasm_interface::Function for $name {
+						fn name(&self) -> &str {
+							stringify!($name)
+						}
+						fn signature(&self) -> $crate::wasm_interface::Signature {
+							gen_signature!( ( $( $params ),* ) $( -> $returns )? )
+						}
+						fn execute(
+							&self,
+							context: &mut dyn $crate::wasm_interface::FunctionContext,
+							args: &mut dyn Iterator<Item=$crate::wasm_interface::Value>,
+						) -> ::std::result::Result<Option<$crate::wasm_interface::Value>, String> {
+							let mut $context = context;
+							marshall! {
+								args,
+								( $( $names : $params ),* ) $( -> $returns )? => { $( $body )* }
+							}
+						}
+					}
+
+					&$name as &dyn $crate::wasm_interface::Function
+				},
+			}
+			$context,
+			$( $tail )*
+		}
+	);
+
+	( $context:ident, $( $tail:tt )* ) => (
+		gen_functions!(@INTERNAL {} $context, $($tail)*);
+	);
+}
+
+/// Converts the list of arguments coming from WASM into their native types.
+#[macro_export]
+macro_rules! unmarshall_args {
+	( $body:tt, $args_iter:ident, $( $names:ident : $params:ty ),*) => ({
+		$(
+			let $names : $params =
+				$args_iter.next()
+					.and_then(|val| <$params as $crate::wasm_interface::TryFromValue>::try_from_value(val))
+					.expect(
+						"`$args_iter` comes from an argument of Externals::execute_function;
+						args to an external call always matches the signature of the external;
+						external signatures are built with count and types and in order defined by `$params`;
+						here, we iterating on `$params`;
+						qed;
+						"
+					);
+		)*
+		$body
+	})
+}
+
+/// Since we can't specify the type of closure directly at binding site:
+///
+/// ```nocompile
+/// let f: FnOnce() -> Result<<u32 as ConvertibleToWasm>::NativeType, _> = || { /* ... */ };
+/// ```
+///
+/// we use this function to constrain the type of the closure.
+#[inline(always)]
+pub fn constrain_closure<R, F>(f: F) -> F
+where
+	F: FnOnce() -> Result<R, String>
+{
+	f
+}
+
+/// Pass the list of parameters by converting them to respective WASM types.
+#[macro_export]
+macro_rules! marshall {
+	( $args_iter:ident, ( $( $names:ident : $params:ty ),* ) -> $returns:ty => $body:tt ) => ({
+		let body = $crate::wasm_utils::constrain_closure::<$returns, _>(|| {
+			unmarshall_args!($body, $args_iter, $( $names : $params ),*)
+		});
+		let r = body()?;
+		return Ok(Some($crate::wasm_interface::IntoValue::into_value(r)))
+	});
+	( $args_iter:ident, ( $( $names:ident : $params:ty ),* ) => $body:tt ) => ({
+		let body = $crate::wasm_utils::constrain_closure::<(), _>(|| {
+			unmarshall_args!($body, $args_iter, $( $names : $params ),*)
+		});
+		body()?;
+		return Ok(None)
+	})
+}
+
+/// Implements the wasm host interface for the given type.
+#[macro_export]
+macro_rules! impl_wasm_host_interface {
+	(
+		impl $interface_name:ident where $context:ident {
+			$(
+				$name:ident($( $names:ident : $params:ty ),* $(,)? ) $( -> $returns:ty )?
+				{ $( $body:tt )* }
+			)*
+		}
+	) => (
+		impl $crate::wasm_interface::HostFunctions for $interface_name {
+			#[allow(non_camel_case_types)]
+			fn host_functions() -> Vec<&'static dyn $crate::wasm_interface::Function> {
+				gen_functions!(
+					$context,
+					$( $name( $( $names: $params ),* ) $( -> $returns )? { $( $body )* } )*
+				)
+			}
+		}
+	);
+}
+
+/// Runtime API functions return an i64 which encodes a pointer in the least-significant 32 bits
+/// and a length in the most-significant 32 bits. This interprets the returned value as a pointer,
+/// length tuple.
+pub fn interpret_runtime_api_result(retval: i64) -> (Pointer<u8>, WordSize) {
+	let ptr = <Pointer<u8>>::new(retval as u32);
+	// The first cast to u64 is necessary so that the right shift does not sign-extend.
+	let len = ((retval as u64) >> 32) as WordSize;
+	(ptr, len)
+}
+

substrate/client/executor/src/wasmi_execution.rs

@@ -0,0 +1,612 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Implementation of a Wasm runtime using the Wasmi interpreter.
+
+use std::{str, mem};
+use wasmi::{
+	Module, ModuleInstance, MemoryInstance, MemoryRef, TableRef, ImportsBuilder, ModuleRef,
+	memory_units::Pages, RuntimeValue::{I32, I64, self},
+};
+use crate::error::{Error, WasmError};
+use codec::{Encode, Decode};
+use primitives::{sandbox as sandbox_primitives, traits::Externalities};
+use crate::sandbox;
+use crate::allocator;
+use crate::wasm_utils::interpret_runtime_api_result;
+use crate::wasm_runtime::WasmRuntime;
+use log::{error, trace};
+use parity_wasm::elements::{deserialize_buffer, DataSegment, Instruction, Module as RawModule};
+use wasm_interface::{
+	FunctionContext, Pointer, WordSize, Sandbox, MemoryId, Result as WResult, Function,
+};
+
+struct FunctionExecutor<'a> {
+	sandbox_store: sandbox::Store<wasmi::FuncRef>,
+	heap: allocator::FreeingBumpHeapAllocator,
+	memory: MemoryRef,
+	table: Option<TableRef>,
+	host_functions: &'a [&'static dyn Function],
+}
+
+impl<'a> FunctionExecutor<'a> {
+	fn new(
+		m: MemoryRef,
+		heap_base: u32,
+		t: Option<TableRef>,
+		host_functions: &'a [&'static dyn Function],
+	) -> Result<Self, Error> {
+		Ok(FunctionExecutor {
+			sandbox_store: sandbox::Store::new(),
+			heap: allocator::FreeingBumpHeapAllocator::new(heap_base),
+			memory: m,
+			table: t,
+			host_functions,
+		})
+	}
+}
+
+impl<'a> sandbox::SandboxCapabilities for FunctionExecutor<'a> {
+	type SupervisorFuncRef = wasmi::FuncRef;
+
+	fn store(&self) -> &sandbox::Store<Self::SupervisorFuncRef> {
+		&self.sandbox_store
+	}
+	fn store_mut(&mut self) -> &mut sandbox::Store<Self::SupervisorFuncRef> {
+		&mut self.sandbox_store
+	}
+	fn allocate(&mut self, len: WordSize) -> Result<Pointer<u8>, Error> {
+		let heap = &mut self.heap;
+		self.memory.with_direct_access_mut(|mem| {
+			heap.allocate(mem, len)
+		})
+	}
+	fn deallocate(&mut self, ptr: Pointer<u8>) -> Result<(), Error> {
+		let heap = &mut self.heap;
+		self.memory.with_direct_access_mut(|mem| {
+			heap.deallocate(mem, ptr)
+		})
+	}
+	fn write_memory(&mut self, ptr: Pointer<u8>, data: &[u8]) -> Result<(), Error> {
+		self.memory.set(ptr.into(), data).map_err(Into::into)
+	}
+	fn read_memory(&self, ptr: Pointer<u8>, len: WordSize) -> Result<Vec<u8>, Error> {
+		self.memory.get(ptr.into(), len as usize).map_err(Into::into)
+	}
+
+	fn invoke(
+		&mut self,
+		dispatch_thunk: &Self::SupervisorFuncRef,
+		invoke_args_ptr: Pointer<u8>,
+		invoke_args_len: WordSize,
+		state: u32,
+		func_idx: sandbox::SupervisorFuncIndex,
+	) -> Result<i64, Error>
+	{
+		let result = wasmi::FuncInstance::invoke(
+			dispatch_thunk,
+			&[
+				RuntimeValue::I32(u32::from(invoke_args_ptr) as i32),
+				RuntimeValue::I32(invoke_args_len as i32),
+				RuntimeValue::I32(state as i32),
+				RuntimeValue::I32(usize::from(func_idx) as i32),
+			],
+			self,
+		);
+		match result {
+			Ok(Some(RuntimeValue::I64(val))) => Ok(val),
+			Ok(_) => return Err("Supervisor function returned unexpected result!".into()),
+			Err(err) => Err(Error::Trap(err)),
+		}
+	}
+}
+
+impl<'a> FunctionContext for FunctionExecutor<'a> {
+	fn read_memory_into(&self, address: Pointer<u8>, dest: &mut [u8]) -> WResult<()> {
+		self.memory.get_into(address.into(), dest).map_err(|e| e.to_string())
+	}
+
+	fn write_memory(&mut self, address: Pointer<u8>, data: &[u8]) -> WResult<()> {
+		self.memory.set(address.into(), data).map_err(|e| e.to_string())
+	}
+
+	fn allocate_memory(&mut self, size: WordSize) -> WResult<Pointer<u8>> {
+		let heap = &mut self.heap;
+		self.memory.with_direct_access_mut(|mem| {
+			heap.allocate(mem, size).map_err(|e| e.to_string())
+		})
+	}
+
+	fn deallocate_memory(&mut self, ptr: Pointer<u8>) -> WResult<()> {
+		let heap = &mut self.heap;
+		self.memory.with_direct_access_mut(|mem| {
+			heap.deallocate(mem, ptr).map_err(|e| e.to_string())
+		})
+	}
+
+	fn sandbox(&mut self) -> &mut dyn Sandbox {
+		self
+	}
+}
+
+impl<'a> Sandbox for FunctionExecutor<'a> {
+	fn memory_get(
+		&mut self,
+		memory_id: MemoryId,
+		offset: WordSize,
+		buf_ptr: Pointer<u8>,
+		buf_len: WordSize,
+	) -> WResult<u32> {
+		let sandboxed_memory = self.sandbox_store.memory(memory_id).map_err(|e| e.to_string())?;
+
+		match MemoryInstance::transfer(
+			&sandboxed_memory,
+			offset as usize,
+			&self.memory,
+			buf_ptr.into(),
+			buf_len as usize,
+		) {
+			Ok(()) => Ok(sandbox_primitives::ERR_OK),
+			Err(_) => Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS),
+		}
+	}
+
+	fn memory_set(
+		&mut self,
+		memory_id: MemoryId,
+		offset: WordSize,
+		val_ptr: Pointer<u8>,
+		val_len: WordSize,
+	) -> WResult<u32> {
+		let sandboxed_memory = self.sandbox_store.memory(memory_id).map_err(|e| e.to_string())?;
+
+		match MemoryInstance::transfer(
+			&self.memory,
+			val_ptr.into(),
+			&sandboxed_memory,
+			offset as usize,
+			val_len as usize,
+		) {
+			Ok(()) => Ok(sandbox_primitives::ERR_OK),
+			Err(_) => Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS),
+		}
+	}
+
+	fn memory_teardown(&mut self, memory_id: MemoryId) -> WResult<()> {
+		self.sandbox_store.memory_teardown(memory_id).map_err(|e| e.to_string())
+	}
+
+	fn memory_new(
+		&mut self,
+		initial: u32,
+		maximum: u32,
+	) -> WResult<MemoryId> {
+		self.sandbox_store.new_memory(initial, maximum).map_err(|e| e.to_string())
+	}
+
+	fn invoke(
+		&mut self,
+		instance_id: u32,
+		export_name: &str,
+		args: &[u8],
+		return_val: Pointer<u8>,
+		return_val_len: WordSize,
+		state: u32,
+	) -> WResult<u32> {
+		trace!(target: "sr-sandbox", "invoke, instance_idx={}", instance_id);
+
+		// Deserialize arguments and convert them into wasmi types.
+		let args = Vec::<sandbox_primitives::TypedValue>::decode(&mut &args[..])
+			.map_err(|_| "Can't decode serialized arguments for the invocation")?
+			.into_iter()
+			.map(Into::into)
+			.collect::<Vec<_>>();
+
+		let instance = self.sandbox_store.instance(instance_id).map_err(|e| e.to_string())?;
+		let result = instance.invoke(export_name, &args, self, state);
+
+		match result {
+			Ok(None) => Ok(sandbox_primitives::ERR_OK),
+			Ok(Some(val)) => {
+				// Serialize return value and write it back into the memory.
+				sandbox_primitives::ReturnValue::Value(val.into()).using_encoded(|val| {
+					if val.len() > return_val_len as usize {
+						Err("Return value buffer is too small")?;
+					}
+					self.write_memory(return_val, val).map_err(|_| "Return value buffer is OOB")?;
+					Ok(sandbox_primitives::ERR_OK)
+				})
+			}
+			Err(_) => Ok(sandbox_primitives::ERR_EXECUTION),
+		}
+	}
+
+	fn instance_teardown(&mut self, instance_id: u32) -> WResult<()> {
+		self.sandbox_store.instance_teardown(instance_id).map_err(|e| e.to_string())
+	}
+
+	fn instance_new(
+		&mut self,
+		dispatch_thunk_id: u32,
+		wasm: &[u8],
+		raw_env_def: &[u8],
+		state: u32,
+	) -> WResult<u32> {
+		// Extract a dispatch thunk from instance's table by the specified index.
+		let dispatch_thunk = {
+			let table = self.table.as_ref()
+				.ok_or_else(|| "Runtime doesn't have a table; sandbox is unavailable")?;
+			table.get(dispatch_thunk_id)
+				.map_err(|_| "dispatch_thunk_idx is out of the table bounds")?
+				.ok_or_else(|| "dispatch_thunk_idx points on an empty table entry")?
+				.clone()
+		};
+
+		let instance_idx_or_err_code =
+			match sandbox::instantiate(self, dispatch_thunk, wasm, raw_env_def, state) {
+				Ok(instance_idx) => instance_idx,
+				Err(sandbox::InstantiationError::StartTrapped) =>
+					sandbox_primitives::ERR_EXECUTION,
+				Err(_) => sandbox_primitives::ERR_MODULE,
+			};
+
+		Ok(instance_idx_or_err_code as u32)
+	}
+}
+
+struct Resolver<'a>(&'a[&'static dyn Function]);
+
+impl<'a> wasmi::ModuleImportResolver for Resolver<'a> {
+	fn resolve_func(&self, name: &str, signature: &wasmi::Signature)
+		-> std::result::Result<wasmi::FuncRef, wasmi::Error>
+	{
+		let signature = wasm_interface::Signature::from(signature);
+		for (function_index, function) in self.0.iter().enumerate() {
+			if name == function.name() {
+				if signature == function.signature() {
+					return Ok(
+						wasmi::FuncInstance::alloc_host(signature.into(), function_index),
+					)
+				} else {
+					return Err(wasmi::Error::Instantiation(
+						format!(
+							"Invalid signature for function `{}` expected `{:?}`, got `{:?}`",
+							function.name(),
+							signature,
+							function.signature(),
+						),
+					))
+				}
+			}
+		}
+
+		Err(wasmi::Error::Instantiation(
+			format!("Export {} not found", name),
+		))
+	}
+}
+
+impl<'a> wasmi::Externals for FunctionExecutor<'a> {
+	fn invoke_index(&mut self, index: usize, args: wasmi::RuntimeArgs)
+		-> Result<Option<wasmi::RuntimeValue>, wasmi::Trap>
+	{
+		let mut args = args.as_ref().iter().copied().map(Into::into);
+		let function = self.host_functions.get(index).ok_or_else(||
+			Error::from(
+				format!("Could not find host function with index: {}", index),
+			)
+		)?;
+
+		function.execute(self, &mut args)
+			.map_err(|msg| Error::FunctionExecution(function.name().to_string(), msg))
+			.map_err(wasmi::Trap::from)
+			.map(|v| v.map(Into::into))
+	}
+}
+
+fn get_mem_instance(module: &ModuleRef) -> Result<MemoryRef, Error> {
+	Ok(module
+		.export_by_name("memory")
+		.ok_or_else(|| Error::InvalidMemoryReference)?
+		.as_memory()
+		.ok_or_else(|| Error::InvalidMemoryReference)?
+		.clone())
+}
+
+/// Find the global named `__heap_base` in the given wasm module instance and
+/// tries to get its value.
+fn get_heap_base(module: &ModuleRef) -> Result<u32, Error> {
+	let heap_base_val = module
+		.export_by_name("__heap_base")
+		.ok_or_else(|| Error::HeapBaseNotFoundOrInvalid)?
+		.as_global()
+		.ok_or_else(|| Error::HeapBaseNotFoundOrInvalid)?
+		.get();
+
+	match heap_base_val {
+		wasmi::RuntimeValue::I32(v) => Ok(v as u32),
+		_ => Err(Error::HeapBaseNotFoundOrInvalid),
+	}
+}
+
+/// Call a given method in the given wasm-module runtime.
+fn call_in_wasm_module(
+	ext: &mut dyn Externalities,
+	module_instance: &ModuleRef,
+	method: &str,
+	data: &[u8],
+	host_functions: &[&'static dyn Function],
+) -> Result<Vec<u8>, Error> {
+	// extract a reference to a linear memory, optional reference to a table
+	// and then initialize FunctionExecutor.
+	let memory = get_mem_instance(module_instance)?;
+	let table: Option<TableRef> = module_instance
+		.export_by_name("__indirect_function_table")
+		.and_then(|e| e.as_table().cloned());
+	let heap_base = get_heap_base(module_instance)?;
+
+	let mut fec = FunctionExecutor::new(memory.clone(), heap_base, table, host_functions)?;
+
+	// Write the call data
+	let offset = fec.allocate_memory(data.len() as u32)?;
+	fec.write_memory(offset, data)?;
+
+	let result = externalities::set_and_run_with_externalities(
+		ext,
+		|| module_instance.invoke_export(
+			method,
+			&[I32(u32::from(offset) as i32), I32(data.len() as i32)],
+			&mut fec,
+		),
+	);
+
+	match result {
+		Ok(Some(I64(r))) => {
+			let (ptr, length) = interpret_runtime_api_result(r);
+			memory.get(ptr.into(), length as usize).map_err(|_| Error::Runtime)
+		},
+		Err(e) => {
+			trace!(
+				target: "wasm-executor",
+				"Failed to execute code with {} pages",
+				memory.current_size().0
+			);
+			Err(e.into())
+		},
+		_ => Err(Error::InvalidReturn),
+	}
+}
+
+/// Prepare module instance
+fn instantiate_module(
+	heap_pages: usize,
+	module: &Module,
+	host_functions: &[&'static dyn Function],
+) -> Result<ModuleRef, Error> {
+	let resolver = Resolver(host_functions);
+	// start module instantiation. Don't run 'start' function yet.
+	let intermediate_instance = ModuleInstance::new(
+		module,
+		&ImportsBuilder::new().with_resolver("env", &resolver),
+	)?;
+
+	// Verify that the module has the heap base global variable.
+	let _ = get_heap_base(intermediate_instance.not_started_instance())?;
+
+	// Extract a reference to a linear memory.
+	let memory = get_mem_instance(intermediate_instance.not_started_instance())?;
+	memory.grow(Pages(heap_pages)).map_err(|_| Error::Runtime)?;
+
+	if intermediate_instance.has_start() {
+		// Runtime is not allowed to have the `start` function.
+		Err(Error::RuntimeHasStartFn)
+	} else {
+		Ok(intermediate_instance.assert_no_start())
+	}
+}
+
+/// A state snapshot of an instance taken just after instantiation.
+///
+/// It is used for restoring the state of the module after execution.
+#[derive(Clone)]
+struct StateSnapshot {
+	/// The offset and the content of the memory segments that should be used to restore the snapshot
+	data_segments: Vec<(u32, Vec<u8>)>,
+	/// The list of all global mutable variables of the module in their sequential order.
+	global_mut_values: Vec<RuntimeValue>,
+	heap_pages: u64,
+}
+
+impl StateSnapshot {
+	// Returns `None` if instance is not valid.
+	fn take(
+		module_instance: &ModuleRef,
+		data_segments: Vec<DataSegment>,
+		heap_pages: u64,
+	) -> Option<Self> {
+		let prepared_segments = data_segments
+			.into_iter()
+			.map(|mut segment| {
+				// Just replace contents of the segment since the segments will be discarded later
+				// anyway.
+				let contents = mem::replace(segment.value_mut(), vec![]);
+
+				let init_expr = match segment.offset() {
+					Some(offset) => offset.code(),
+					// Return if the segment is passive
+					None => return None
+				};
+
+				// [op, End]
+				if init_expr.len() != 2 {
+					return None;
+				}
+				let offset = match init_expr[0] {
+					Instruction::I32Const(v) => v as u32,
+					Instruction::GetGlobal(idx) => {
+						let global_val = module_instance.globals().get(idx as usize)?.get();
+						match global_val {
+							RuntimeValue::I32(v) => v as u32,
+							_ => return None,
+						}
+					}
+					_ => return None,
+				};
+
+				Some((offset, contents))
+			})
+			.collect::<Option<Vec<_>>>()?;
+
+		// Collect all values of mutable globals.
+		let global_mut_values = module_instance
+			.globals()
+			.iter()
+			.filter(|g| g.is_mutable())
+			.map(|g| g.get())
+			.collect();
+
+		Some(Self {
+			data_segments: prepared_segments,
+			global_mut_values,
+			heap_pages,
+		})
+	}
+
+	/// Reset the runtime instance to the initial version by restoring
+	/// the preserved memory and globals.
+	///
+	/// Returns `Err` if applying the snapshot is failed.
+	fn apply(&self, instance: &ModuleRef) -> Result<(), WasmError> {
+		let memory = instance
+			.export_by_name("memory")
+			.ok_or(WasmError::ApplySnapshotFailed)?
+			.as_memory()
+			.cloned()
+			.ok_or(WasmError::ApplySnapshotFailed)?;
+
+		// First, erase the memory and copy the data segments into it.
+		memory
+			.erase()
+			.map_err(|e| WasmError::ErasingFailed(e.to_string()))?;
+		for (offset, contents) in &self.data_segments {
+			memory
+				.set(*offset, contents)
+				.map_err(|_| WasmError::ApplySnapshotFailed)?;
+		}
+
+		// Second, restore the values of mutable globals.
+		for (global_ref, global_val) in instance
+			.globals()
+			.iter()
+			.filter(|g| g.is_mutable())
+			.zip(self.global_mut_values.iter())
+			{
+				// the instance should be the same as used for preserving and
+				// we iterate the same way it as we do it for preserving values that means that the
+				// types should be the same and all the values are mutable. So no error is expected/
+				global_ref
+					.set(*global_val)
+					.map_err(|_| WasmError::ApplySnapshotFailed)?;
+			}
+		Ok(())
+	}
+}
+
+/// A runtime along with its initial state snapshot.
+#[derive(Clone)]
+pub struct WasmiRuntime {
+	/// A wasm module instance.
+	instance: ModuleRef,
+	/// The snapshot of the instance's state taken just after the instantiation.
+	state_snapshot: StateSnapshot,
+	/// The host functions registered for this instance.
+	host_functions: Vec<&'static dyn Function>,
+}
+
+impl WasmRuntime for WasmiRuntime {
+	fn update_heap_pages(&mut self, heap_pages: u64) -> bool {
+		self.state_snapshot.heap_pages == heap_pages
+	}
+
+	fn host_functions(&self) -> &[&'static dyn Function] {
+		&self.host_functions
+	}
+
+	fn call(
+		&mut self,
+		ext: &mut dyn Externalities,
+		method: &str,
+		data: &[u8],
+	) -> Result<Vec<u8>, Error> {
+		self.state_snapshot.apply(&self.instance)
+			.map_err(|e| {
+				// Snapshot restoration failed. This is pretty unexpected since this can happen
+				// if some invariant is broken or if the system is under extreme memory pressure
+				// (so erasing fails).
+				error!(target: "wasm-executor", "snapshot restoration failed: {}", e);
+				e
+			})?;
+		call_in_wasm_module(ext, &self.instance, method, data, &self.host_functions)
+	}
+}
+
+pub fn create_instance(
+	code: &[u8],
+	heap_pages: u64,
+	host_functions: Vec<&'static dyn Function>,
+) -> Result<WasmiRuntime, WasmError> {
+	let module = Module::from_buffer(&code).map_err(|_| WasmError::InvalidModule)?;
+
+	// Extract the data segments from the wasm code.
+	//
+	// A return of this error actually indicates that there is a problem in logic, since
+	// we just loaded and validated the `module` above.
+	let data_segments = extract_data_segments(&code)?;
+
+	// Instantiate this module.
+	let instance = instantiate_module(heap_pages as usize, &module, &host_functions)
+		.map_err(|e| WasmError::Instantiation(e.to_string()))?;
+
+	// Take state snapshot before executing anything.
+	let state_snapshot = StateSnapshot::take(&instance, data_segments, heap_pages)
+		.expect(
+			"`take` returns `Err` if the module is not valid;
+				we already loaded module above, thus the `Module` is proven to be valid at this point;
+				qed
+				",
+		);
+
+	Ok(WasmiRuntime {
+		instance,
+		state_snapshot,
+		host_functions,
+	})
+}
+
+/// Extract the data segments from the given wasm code.
+///
+/// Returns `Err` if the given wasm code cannot be deserialized.
+fn extract_data_segments(wasm_code: &[u8]) -> Result<Vec<DataSegment>, WasmError> {
+	let raw_module: RawModule = deserialize_buffer(wasm_code)
+		.map_err(|_| WasmError::CantDeserializeWasm)?;
+
+	let segments = raw_module
+		.data_section()
+		.map(|ds| ds.entries())
+		.unwrap_or(&[])
+		.to_vec();
+	Ok(segments)
+}

substrate/client/executor/src/wasmtime/function_executor.rs

@@ -0,0 +1,387 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::allocator::FreeingBumpHeapAllocator;
+use crate::error::{Error, Result};
+use crate::sandbox::{self, SandboxCapabilities, SupervisorFuncIndex};
+use crate::wasmtime::util::{
+	checked_range, cranelift_ir_signature, read_memory_into, write_memory_from,
+};
+
+use codec::{Decode, Encode};
+use cranelift_codegen::ir;
+use cranelift_codegen::isa::TargetFrontendConfig;
+use log::trace;
+use primitives::sandbox as sandbox_primitives;
+use std::{cmp, mem, ptr};
+use wasmtime_environ::translate_signature;
+use wasmtime_jit::{ActionError, Compiler};
+use wasmtime_runtime::{Export, VMCallerCheckedAnyfunc, VMContext, wasmtime_call_trampoline};
+use wasm_interface::{
+	FunctionContext, MemoryId, Pointer, Result as WResult, Sandbox, Signature, Value, ValueType,
+	WordSize,
+};
+
+/// Wrapper type for pointer to a Wasm table entry.
+///
+/// The wrapper type is used to ensure that the function reference is valid as it must be unsafely
+/// dereferenced from within the safe method `<FunctionExecutor as SandboxCapabilities>::invoke`.
+#[derive(Clone, Copy)]
+pub struct SupervisorFuncRef(*const VMCallerCheckedAnyfunc);
+
+/// The state required to construct a FunctionExecutor context. The context only lasts for one host
+/// call, whereas the state is maintained for the duration of a Wasm runtime call, which may make
+/// many different host calls that must share state.
+///
+/// This is stored as part of the host state of the "env" Wasmtime instance.
+pub struct FunctionExecutorState {
+	sandbox_store: sandbox::Store<SupervisorFuncRef>,
+	heap: FreeingBumpHeapAllocator,
+}
+
+impl FunctionExecutorState {
+	/// Constructs a new `FunctionExecutorState`.
+	pub fn new(heap_base: u32) -> Self {
+		FunctionExecutorState {
+			sandbox_store: sandbox::Store::new(),
+			heap: FreeingBumpHeapAllocator::new(heap_base),
+		}
+	}
+
+	/// Returns a mutable reference to the heap allocator.
+	pub fn heap(&mut self) -> &mut FreeingBumpHeapAllocator {
+		&mut self.heap
+	}
+}
+
+/// A `FunctionExecutor` implements `FunctionContext` for making host calls from a Wasmtime
+/// runtime. The `FunctionExecutor` exists only for the lifetime of the call and borrows state from
+/// a longer-living `FunctionExecutorState`.
+pub struct FunctionExecutor<'a> {
+	compiler: &'a mut Compiler,
+	sandbox_store: &'a mut sandbox::Store<SupervisorFuncRef>,
+	heap: &'a mut FreeingBumpHeapAllocator,
+	memory: &'a mut [u8],
+	table: Option<&'a [VMCallerCheckedAnyfunc]>,
+}
+
+impl<'a> FunctionExecutor<'a> {
+	/// Construct a new `FunctionExecutor`.
+	///
+	/// The vmctx MUST come from a call to a function in the "env" module.
+	/// The state MUST be looked up from the host state of the "env" module.
+	pub unsafe fn new(
+		vmctx: *mut VMContext,
+		compiler: &'a mut Compiler,
+		state: &'a mut FunctionExecutorState,
+	) -> Result<Self>
+	{
+		let memory = match (*vmctx).lookup_global_export("memory") {
+			Some(Export::Memory { definition, vmctx: _, memory: _ }) =>
+				std::slice::from_raw_parts_mut(
+					(*definition).base,
+					(*definition).current_length,
+				),
+			_ => return Err(Error::InvalidMemoryReference),
+		};
+		let table = match (*vmctx).lookup_global_export("__indirect_function_table") {
+			Some(Export::Table { definition, vmctx: _, table: _ }) =>
+				Some(std::slice::from_raw_parts(
+					(*definition).base as *const VMCallerCheckedAnyfunc,
+					(*definition).current_elements as usize,
+				)),
+			_ => None,
+		};
+		Ok(FunctionExecutor {
+			compiler,
+			sandbox_store: &mut state.sandbox_store,
+			heap: &mut state.heap,
+			memory,
+			table,
+		})
+	}
+}
+
+impl<'a> SandboxCapabilities for FunctionExecutor<'a> {
+	type SupervisorFuncRef = SupervisorFuncRef;
+
+	fn store(&self) -> &sandbox::Store<Self::SupervisorFuncRef> {
+		&self.sandbox_store
+	}
+
+	fn store_mut(&mut self) -> &mut sandbox::Store<Self::SupervisorFuncRef> {
+		&mut self.sandbox_store
+	}
+
+	fn allocate(&mut self, len: WordSize) -> Result<Pointer<u8>> {
+		self.heap.allocate(self.memory, len)
+	}
+
+	fn deallocate(&mut self, ptr: Pointer<u8>) -> Result<()> {
+		self.heap.deallocate(self.memory, ptr)
+	}
+
+	fn write_memory(&mut self, ptr: Pointer<u8>, data: &[u8]) -> Result<()> {
+		write_memory_from(self.memory, ptr, data)
+	}
+
+	fn read_memory(&self, ptr: Pointer<u8>, len: WordSize) -> Result<Vec<u8>> {
+		let mut output = vec![0; len as usize];
+		read_memory_into(self.memory, ptr, output.as_mut())?;
+		Ok(output)
+	}
+
+	fn invoke(
+		&mut self,
+		dispatch_thunk: &Self::SupervisorFuncRef,
+		invoke_args_ptr: Pointer<u8>,
+		invoke_args_len: WordSize,
+		state: u32,
+		func_idx: SupervisorFuncIndex,
+	) -> Result<i64>
+	{
+		let func_ptr = unsafe { (*dispatch_thunk.0).func_ptr };
+		let vmctx = unsafe { (*dispatch_thunk.0).vmctx };
+
+		// The following code is based on the wasmtime_jit::Context::invoke.
+		let value_size = mem::size_of::<VMInvokeArgument>();
+		let (signature, mut values_vec) = generate_signature_and_args(
+			&[
+				Value::I32(u32::from(invoke_args_ptr) as i32),
+				Value::I32(invoke_args_len as i32),
+				Value::I32(state as i32),
+				Value::I32(usize::from(func_idx) as i32),
+			],
+			Some(ValueType::I64),
+			self.compiler.frontend_config(),
+		);
+
+		// Get the trampoline to call for this function.
+		let exec_code_buf = self.compiler
+			.get_published_trampoline(func_ptr, &signature, value_size)
+			.map_err(ActionError::Setup)
+			.map_err(Error::Wasmtime)?;
+
+		// Call the trampoline.
+		if let Err(message) = unsafe {
+			wasmtime_call_trampoline(
+				vmctx,
+				exec_code_buf,
+				values_vec.as_mut_ptr() as *mut u8,
+			)
+		} {
+			return Err(Error::Other(message));
+		}
+
+		// Load the return value out of `values_vec`.
+		Ok(unsafe { ptr::read(values_vec.as_ptr() as *const i64) })
+	}
+}
+
+impl<'a> FunctionContext for FunctionExecutor<'a> {
+	fn read_memory_into(&self, address: Pointer<u8>, dest: &mut [u8]) -> WResult<()> {
+		read_memory_into(self.memory, address, dest).map_err(|e| e.to_string())
+	}
+
+	fn write_memory(&mut self, address: Pointer<u8>, data: &[u8]) -> WResult<()> {
+		write_memory_from(self.memory, address, data).map_err(|e| e.to_string())
+	}
+
+	fn allocate_memory(&mut self, size: WordSize) -> WResult<Pointer<u8>> {
+		self.heap.allocate(self.memory, size).map_err(|e| e.to_string())
+	}
+
+	fn deallocate_memory(&mut self, ptr: Pointer<u8>) -> WResult<()> {
+		self.heap.deallocate(self.memory, ptr).map_err(|e| e.to_string())
+	}
+
+	fn sandbox(&mut self) -> &mut dyn Sandbox {
+		self
+	}
+}
+
+impl<'a> Sandbox for FunctionExecutor<'a> {
+	fn memory_get(
+		&mut self,
+		memory_id: MemoryId,
+		offset: WordSize,
+		buf_ptr: Pointer<u8>,
+		buf_len: WordSize,
+	) -> WResult<u32>
+	{
+		let sandboxed_memory = self.sandbox_store.memory(memory_id)
+			.map_err(|e| e.to_string())?;
+		sandboxed_memory.with_direct_access(|memory| {
+			let len = buf_len as usize;
+			let src_range = match checked_range(offset as usize, len, memory.len()) {
+				Some(range) => range,
+				None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS),
+			};
+			let dst_range = match checked_range(buf_ptr.into(), len, self.memory.len()) {
+				Some(range) => range,
+				None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS),
+			};
+			&mut self.memory[dst_range].copy_from_slice(&memory[src_range]);
+			Ok(sandbox_primitives::ERR_OK)
+		})
+	}
+
+	fn memory_set(
+		&mut self,
+		memory_id: MemoryId,
+		offset: WordSize,
+		val_ptr: Pointer<u8>,
+		val_len: WordSize,
+	) -> WResult<u32>
+	{
+		let sandboxed_memory = self.sandbox_store.memory(memory_id)
+			.map_err(|e| e.to_string())?;
+		sandboxed_memory.with_direct_access_mut(|memory| {
+			let len = val_len as usize;
+			let src_range = match checked_range(val_ptr.into(), len, self.memory.len()) {
+				Some(range) => range,
+				None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS),
+			};
+			let dst_range = match checked_range(offset as usize, len, memory.len()) {
+				Some(range) => range,
+				None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS),
+			};
+			&mut memory[dst_range].copy_from_slice(&self.memory[src_range]);
+			Ok(sandbox_primitives::ERR_OK)
+		})
+	}
+
+	fn memory_teardown(&mut self, memory_id: MemoryId)
+		-> WResult<()>
+	{
+		self.sandbox_store.memory_teardown(memory_id).map_err(|e| e.to_string())
+	}
+
+	fn memory_new(&mut self, initial: u32, maximum: MemoryId) -> WResult<u32> {
+		self.sandbox_store.new_memory(initial, maximum).map_err(|e| e.to_string())
+	}
+
+	fn invoke(
+		&mut self,
+		instance_id: u32,
+		export_name: &str,
+		args: &[u8],
+		return_val: Pointer<u8>,
+		return_val_len: u32,
+		state: u32,
+	) -> WResult<u32> {
+		trace!(target: "sr-sandbox", "invoke, instance_idx={}", instance_id);
+
+		// Deserialize arguments and convert them into wasmi types.
+		let args = Vec::<sandbox_primitives::TypedValue>::decode(&mut &args[..])
+			.map_err(|_| "Can't decode serialized arguments for the invocation")?
+			.into_iter()
+			.map(Into::into)
+			.collect::<Vec<_>>();
+
+		let instance = self.sandbox_store.instance(instance_id).map_err(|e| e.to_string())?;
+		let result = instance.invoke(export_name, &args, self, state);
+
+		match result {
+			Ok(None) => Ok(sandbox_primitives::ERR_OK),
+			Ok(Some(val)) => {
+				// Serialize return value and write it back into the memory.
+				sandbox_primitives::ReturnValue::Value(val.into()).using_encoded(|val| {
+					if val.len() > return_val_len as usize {
+						Err("Return value buffer is too small")?;
+					}
+					FunctionContext::write_memory(self, return_val, val)?;
+					Ok(sandbox_primitives::ERR_OK)
+				})
+			}
+			Err(_) => Ok(sandbox_primitives::ERR_EXECUTION),
+		}
+	}
+
+	fn instance_teardown(&mut self, instance_id: u32) -> WResult<()> {
+		self.sandbox_store.instance_teardown(instance_id).map_err(|e| e.to_string())
+	}
+
+	fn instance_new(&mut self, dispatch_thunk_id: u32, wasm: &[u8], raw_env_def: &[u8], state: u32)
+		-> WResult<u32>
+	{
+		// Extract a dispatch thunk from instance's table by the specified index.
+		let dispatch_thunk = {
+			let table = self.table.as_ref()
+				.ok_or_else(|| "Runtime doesn't have a table; sandbox is unavailable")?;
+			let func_ref = table.get(dispatch_thunk_id as usize)
+				.ok_or_else(|| "dispatch_thunk_idx is out of the table bounds")?;
+			SupervisorFuncRef(func_ref)
+		};
+
+		let instance_idx_or_err_code =
+			match sandbox::instantiate(self, dispatch_thunk, wasm, raw_env_def, state) {
+				Ok(instance_idx) => instance_idx,
+				Err(sandbox::InstantiationError::StartTrapped) =>
+					sandbox_primitives::ERR_EXECUTION,
+				Err(_) => sandbox_primitives::ERR_MODULE,
+			};
+
+		Ok(instance_idx_or_err_code as u32)
+	}
+}
+
+// The storage for a Wasmtime invocation argument.
+#[derive(Debug, Default, Copy, Clone)]
+#[repr(C, align(8))]
+struct VMInvokeArgument([u8; 8]);
+
+fn generate_signature_and_args(
+	args: &[Value],
+	result_type: Option<ValueType>,
+	frontend_config: TargetFrontendConfig,
+) -> (ir::Signature, Vec<VMInvokeArgument>)
+{
+	// This code is based on the wasmtime_jit::Context::invoke.
+
+	let param_types = args.iter()
+		.map(|arg| arg.value_type())
+		.collect::<Vec<_>>();
+	let signature = translate_signature(
+		cranelift_ir_signature(
+			Signature::new(param_types, result_type),
+			&frontend_config.default_call_conv
+		),
+		frontend_config.pointer_type()
+	);
+
+	let mut values_vec = vec![
+		VMInvokeArgument::default();
+		cmp::max(args.len(), result_type.iter().len())
+	];
+
+	// Store the argument values into `values_vec`.
+	for (index, arg) in args.iter().enumerate() {
+		unsafe {
+			let ptr = values_vec.as_mut_ptr().add(index);
+
+			match arg {
+				Value::I32(x) => ptr::write(ptr as *mut i32, *x),
+				Value::I64(x) => ptr::write(ptr as *mut i64, *x),
+				Value::F32(x) => ptr::write(ptr as *mut u32, *x),
+				Value::F64(x) => ptr::write(ptr as *mut u64, *x),
+			}
+		}
+	}
+
+	(signature, values_vec)
+}
+

substrate/client/executor/src/wasmtime/mod.rs

@@ -0,0 +1,24 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+///! Defines a `WasmRuntime` that uses the Wasmtime JIT to execute.
+
+mod function_executor;
+mod runtime;
+mod trampoline;
+mod util;
+
+pub use runtime::create_instance;

substrate/client/executor/src/wasmtime/runtime.rs

@@ -0,0 +1,372 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Defines the compiled Wasm runtime that uses Wasmtime internally.
+
+use crate::error::{Error, Result, WasmError};
+use crate::wasm_runtime::WasmRuntime;
+use crate::wasm_utils::interpret_runtime_api_result;
+use crate::wasmtime::function_executor::FunctionExecutorState;
+use crate::wasmtime::trampoline::{EnvState, make_trampoline};
+use crate::wasmtime::util::{cranelift_ir_signature, read_memory_into, write_memory_from};
+use crate::Externalities;
+
+use cranelift_codegen::ir;
+use cranelift_codegen::isa::TargetIsa;
+use cranelift_entity::{EntityRef, PrimaryMap};
+use cranelift_frontend::FunctionBuilderContext;
+use cranelift_wasm::DefinedFuncIndex;
+use std::cell::RefCell;
+use std::collections::HashMap;
+use std::convert::TryFrom;
+use std::rc::Rc;
+use wasm_interface::{Pointer, WordSize, Function};
+use wasmtime_environ::{Module, translate_signature};
+use wasmtime_jit::{
+	ActionOutcome, ActionError, CodeMemory, CompilationStrategy, CompiledModule, Compiler, Context,
+	SetupError, RuntimeValue,
+};
+use wasmtime_runtime::{Export, Imports, InstanceHandle, VMFunctionBody};
+
+/// A `WasmRuntime` implementation using the Wasmtime JIT to compile the runtime module to native
+/// and execute the compiled code.
+pub struct WasmtimeRuntime {
+	module: CompiledModule,
+	context: Context,
+	max_heap_pages: Option<u32>,
+	heap_pages: u32,
+	/// The host functions registered for this instance.
+	host_functions: Vec<&'static dyn Function>,
+}
+
+impl WasmRuntime for WasmtimeRuntime {
+	fn update_heap_pages(&mut self, heap_pages: u64) -> bool {
+		match heap_pages_valid(heap_pages, self.max_heap_pages) {
+			Some(heap_pages) => {
+				self.heap_pages = heap_pages;
+				true
+			}
+			None => false,
+		}
+	}
+
+	fn host_functions(&self) -> &[&'static dyn Function] {
+		&self.host_functions
+	}
+
+	fn call(&mut self, ext: &mut dyn Externalities, method: &str, data: &[u8]) -> Result<Vec<u8>> {
+		call_method(
+			&mut self.context,
+			&mut self.module,
+			ext,
+			method,
+			data,
+			self.heap_pages,
+		)
+	}
+}
+
+/// Create a new `WasmtimeRuntime` given the code. This function performs translation from Wasm to
+/// machine code, which can be computationally heavy.
+pub fn create_instance(
+	code: &[u8],
+	heap_pages: u64,
+	host_functions: Vec<&'static dyn Function>,
+) -> std::result::Result<WasmtimeRuntime, WasmError> {
+	let (compiled_module, context) = create_compiled_unit(code, &host_functions)?;
+
+	// Inspect the module for the min and max memory sizes.
+	let (min_memory_size, max_memory_size) = {
+		let module = compiled_module.module_ref();
+		let memory_index = match module.exports.get("memory") {
+			Some(wasmtime_environ::Export::Memory(memory_index)) => *memory_index,
+			_ => return Err(WasmError::InvalidMemory),
+		};
+		let memory_plan = module.memory_plans.get(memory_index)
+			.expect("memory_index is retrieved from the module's exports map; qed");
+		(memory_plan.memory.minimum, memory_plan.memory.maximum)
+	};
+
+	// Check that heap_pages is within the allowed range.
+	let max_heap_pages = max_memory_size.map(|max| max.saturating_sub(min_memory_size));
+	let heap_pages = heap_pages_valid(heap_pages, max_heap_pages)
+		.ok_or_else(|| WasmError::InvalidHeapPages)?;
+
+	Ok(WasmtimeRuntime {
+		module: compiled_module,
+		context,
+		max_heap_pages,
+		heap_pages,
+		host_functions,
+	})
+}
+
+fn create_compiled_unit(
+	code: &[u8],
+	host_functions: &[&'static dyn Function],
+) -> std::result::Result<(CompiledModule, Context), WasmError> {
+	let compilation_strategy = CompilationStrategy::Cranelift;
+
+	let compiler = new_compiler(compilation_strategy)?;
+	let mut context = Context::new(Box::new(compiler));
+
+	// Enable/disable producing of debug info.
+	context.set_debug_info(false);
+
+	// Instantiate and link the env module.
+	let global_exports = context.get_global_exports();
+	let compiler = new_compiler(compilation_strategy)?;
+	let env_module = instantiate_env_module(global_exports, compiler, host_functions)?;
+	context.name_instance("env".to_owned(), env_module);
+
+	// Compile the wasm module.
+	let module = context.compile_module(&code)
+		.map_err(WasmError::WasmtimeSetup)?;
+
+	Ok((module, context))
+}
+
+/// Call a function inside a precompiled Wasm module.
+fn call_method(
+	context: &mut Context,
+	module: &mut CompiledModule,
+	ext: &mut dyn Externalities,
+	method: &str,
+	data: &[u8],
+	heap_pages: u32,
+) -> Result<Vec<u8>> {
+	// Old exports get clobbered in `InstanceHandle::new` if we don't explicitly remove them first.
+	//
+	// The global exports mechanism is temporary in Wasmtime and expected to be removed.
+	// https://github.com/CraneStation/wasmtime/issues/332
+	clear_globals(&mut *context.get_global_exports().borrow_mut());
+
+	let mut instance = module.instantiate()
+		.map_err(SetupError::Instantiate)
+		.map_err(ActionError::Setup)
+		.map_err(Error::Wasmtime)?;
+
+	// Ideally there would be a way to set the heap pages during instantiation rather than
+	// growing the memory after the fact. Currently this may require an additional mmap and copy.
+	// However, the wasmtime API doesn't support modifying the size of memory on instantiation
+	// at this time.
+	grow_memory(&mut instance, heap_pages)?;
+
+	// Initialize the function executor state.
+	let heap_base = get_heap_base(&instance)?;
+	let executor_state = FunctionExecutorState::new(heap_base);
+	reset_env_state_and_take_trap(context, Some(executor_state))?;
+
+	// Write the input data into guest memory.
+	let (data_ptr, data_len) = inject_input_data(context, &mut instance, data)?;
+	let args = [RuntimeValue::I32(u32::from(data_ptr) as i32), RuntimeValue::I32(data_len as i32)];
+
+	// Invoke the function in the runtime.
+	let outcome = externalities::set_and_run_with_externalities(ext, || {
+		context
+			.invoke(&mut instance, method, &args[..])
+			.map_err(Error::Wasmtime)
+	})?;
+	let trap_error = reset_env_state_and_take_trap(context, None)?;
+	let (output_ptr, output_len) = match outcome {
+		ActionOutcome::Returned { values } => match values.as_slice() {
+			[RuntimeValue::I64(retval)] => interpret_runtime_api_result(*retval),
+			_ => return Err(Error::InvalidReturn),
+		}
+		ActionOutcome::Trapped { message } => return Err(trap_error.unwrap_or_else(
+			|| format!("Wasm execution trapped: {}", message).into()
+		)),
+	};
+
+	// Read the output data from guest memory.
+	let mut output = vec![0; output_len as usize];
+	let memory = get_memory_mut(&mut instance)?;
+	read_memory_into(memory, output_ptr, &mut output)?;
+	Ok(output)
+}
+
+/// The implementation is based on wasmtime_wasi::instantiate_wasi.
+fn instantiate_env_module(
+	global_exports: Rc<RefCell<HashMap<String, Option<Export>>>>,
+	compiler: Compiler,
+	host_functions: &[&'static dyn Function],
+) -> std::result::Result<InstanceHandle, WasmError>
+{
+	let isa = target_isa()?;
+	let pointer_type = isa.pointer_type();
+	let call_conv = isa.default_call_conv();
+
+	let mut fn_builder_ctx = FunctionBuilderContext::new();
+	let mut module = Module::new();
+	let mut finished_functions = <PrimaryMap<DefinedFuncIndex, *const VMFunctionBody>>::new();
+	let mut code_memory = CodeMemory::new();
+
+	for function in host_functions {
+		let sig = translate_signature(
+			cranelift_ir_signature(function.signature(), &call_conv),
+			pointer_type
+		);
+		let sig_id = module.signatures.push(sig.clone());
+		let func_id = module.functions.push(sig_id);
+		module
+			.exports
+			.insert(function.name().to_string(), wasmtime_environ::Export::Function(func_id));
+
+		let trampoline = make_trampoline(
+			isa.as_ref(),
+			&mut code_memory,
+			&mut fn_builder_ctx,
+			func_id.index() as u32,
+			&sig,
+		)?;
+		finished_functions.push(trampoline);
+	}
+
+	code_memory.publish();
+
+	let imports = Imports::none();
+	let data_initializers = Vec::new();
+	let signatures = PrimaryMap::new();
+	let env_state = EnvState::new(code_memory, compiler, host_functions);
+
+	let result = InstanceHandle::new(
+		Rc::new(module),
+		global_exports,
+		finished_functions.into_boxed_slice(),
+		imports,
+		&data_initializers,
+		signatures.into_boxed_slice(),
+		None,
+		Box::new(env_state),
+	);
+	result.map_err(|e| WasmError::WasmtimeSetup(SetupError::Instantiate(e)))
+}
+
+/// Build a new TargetIsa for the host machine.
+fn target_isa() -> std::result::Result<Box<dyn TargetIsa>, WasmError> {
+	let isa_builder = cranelift_native::builder()
+		.map_err(WasmError::MissingCompilerSupport)?;
+	let flag_builder = cranelift_codegen::settings::builder();
+	Ok(isa_builder.finish(cranelift_codegen::settings::Flags::new(flag_builder)))
+}
+
+fn new_compiler(strategy: CompilationStrategy) -> std::result::Result<Compiler, WasmError> {
+	let isa = target_isa()?;
+	Ok(Compiler::new(isa, strategy))
+}
+
+fn clear_globals(global_exports: &mut HashMap<String, Option<Export>>) {
+	global_exports.remove("memory");
+	global_exports.remove("__heap_base");
+	global_exports.remove("__indirect_function_table");
+}
+
+fn grow_memory(instance: &mut InstanceHandle, pages: u32) -> Result<()> {
+	// This is safe to wrap in an unsafe block as:
+	// - The result of the `lookup_immutable` call is not mutated
+	// - The definition pointer is returned by a lookup on a valid instance
+	let memory_index = unsafe {
+		match instance.lookup_immutable("memory") {
+			Some(Export::Memory { definition, vmctx: _, memory: _ }) =>
+				instance.memory_index(&*definition),
+			_ => return Err(Error::InvalidMemoryReference),
+		}
+	};
+	instance.memory_grow(memory_index, pages)
+		.map(|_| ())
+		.ok_or_else(|| "requested heap_pages would exceed maximum memory size".into())
+}
+
+fn get_env_state(context: &mut Context) -> Result<&mut EnvState> {
+	let env_instance = context.get_instance("env")
+		.map_err(|err| format!("cannot find \"env\" module: {}", err))?;
+	env_instance
+		.host_state()
+		.downcast_mut::<EnvState>()
+		.ok_or_else(|| "cannot get \"env\" module host state".into())
+}
+
+fn reset_env_state_and_take_trap(
+	context: &mut Context,
+	executor_state: Option<FunctionExecutorState>,
+) -> Result<Option<Error>>
+{
+	let env_state = get_env_state(context)?;
+	env_state.executor_state = executor_state;
+	Ok(env_state.take_trap())
+}
+
+fn inject_input_data(
+	context: &mut Context,
+	instance: &mut InstanceHandle,
+	data: &[u8],
+) -> Result<(Pointer<u8>, WordSize)> {
+	let env_state = get_env_state(context)?;
+	let executor_state = env_state.executor_state
+		.as_mut()
+		.ok_or_else(|| "cannot get \"env\" module executor state")?;
+
+	let memory = get_memory_mut(instance)?;
+
+	let data_len = data.len() as WordSize;
+	let data_ptr = executor_state.heap().allocate(memory, data_len)?;
+	write_memory_from(memory, data_ptr, data)?;
+	Ok((data_ptr, data_len))
+}
+
+fn get_memory_mut(instance: &mut InstanceHandle) -> Result<&mut [u8]> {
+	match instance.lookup("memory") {
+		// This is safe to wrap in an unsafe block as:
+		// - The definition pointer is returned by a lookup on a valid instance and thus points to
+		//   a valid memory definition
+		Some(Export::Memory { definition, vmctx: _, memory: _ }) => unsafe {
+			Ok(std::slice::from_raw_parts_mut(
+				(*definition).base,
+				(*definition).current_length,
+			))
+		},
+		_ => Err(Error::InvalidMemoryReference),
+	}
+}
+
+fn get_heap_base(instance: &InstanceHandle) -> Result<u32> {
+	// This is safe to wrap in an unsafe block as:
+	// - The result of the `lookup_immutable` call is not mutated
+	// - The definition pointer is returned by a lookup on a valid instance
+	// - The defined value is checked to be an I32, which can be read safely as a u32
+	unsafe {
+		match instance.lookup_immutable("__heap_base") {
+			Some(Export::Global { definition, vmctx: _, global })
+				if global.ty == ir::types::I32 =>
+				Ok(*(*definition).as_u32()),
+			_ => return Err(Error::HeapBaseNotFoundOrInvalid),
+		}
+	}
+}
+
+/// Checks whether the heap_pages parameter is within the valid range and converts it to a u32.
+/// Returns None if heaps_pages in not in range.
+fn heap_pages_valid(heap_pages: u64, max_heap_pages: Option<u32>)
+	-> Option<u32>
+{
+	let heap_pages = u32::try_from(heap_pages).ok()?;
+	if let Some(max_heap_pages) = max_heap_pages {
+		if heap_pages > max_heap_pages {
+			return None;
+		}
+	}
+	Some(heap_pages)
+}

substrate/client/executor/src/wasmtime/trampoline.rs

@@ -0,0 +1,347 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The trampoline is the dynamically generated entry point to a runtime host call.
+//!
+//! This code is based on and large parts are copied from wasmtime's
+//! wasmtime-api/src/trampoline/func.rs.
+
+use cranelift_codegen::{Context, binemit, ir, isa};
+use cranelift_codegen::ir::{InstBuilder, StackSlotData, StackSlotKind, TrapCode};
+use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
+use wasmtime_jit::{CodeMemory, Compiler};
+use wasmtime_runtime::{VMContext, VMFunctionBody};
+use wasm_interface::{Function, Value, ValueType};
+use std::{cmp, panic::{self, AssertUnwindSafe}, ptr};
+
+use crate::error::{Error, WasmError};
+use crate::wasmtime::function_executor::{FunctionExecutorState, FunctionExecutor};
+
+const CALL_SUCCESS: u32 = 0;
+const CALL_FAILED_WITH_ERROR: u32 = 1;
+const CALL_WITH_BAD_HOST_STATE: u32 = 2;
+
+/// A code to trap with that indicates a host call error.
+const TRAP_USER_CODE: u16 = 0;
+
+/// The only Wasm types allowed in host function signatures (I32, I64, F32, F64) are all
+/// represented in at most 8 bytes.
+const MAX_WASM_TYPE_SIZE: usize = 8;
+
+/// The top-level host state of the "env" module. This state is used by the trampoline function to
+/// construct a `FunctionExecutor` which can execute the host call.
+pub struct EnvState {
+	host_functions: Vec<&'static dyn Function>,
+	compiler: Compiler,
+	// The code memory must be kept around on the state to prevent it from being dropped.
+	#[allow(dead_code)]
+	code_memory: CodeMemory,
+	trap: Option<Error>,
+	/// The executor state stored across host calls during a single Wasm runtime call.
+	/// During a runtime call, this MUST be `Some`.
+	pub executor_state: Option<FunctionExecutorState>,
+}
+
+impl EnvState {
+	/// Construct a new `EnvState` which owns the given code memory.
+	pub fn new(
+		code_memory: CodeMemory,
+		compiler: Compiler,
+		host_functions: &[&'static dyn Function],
+	) -> Self {
+		EnvState {
+			trap: None,
+			compiler,
+			code_memory,
+			executor_state: None,
+			host_functions: host_functions.to_vec(),
+		}
+	}
+
+	/// Resets the trap error to None and returns the current value.
+	pub fn take_trap(&mut self) -> Option<Error> {
+		self.trap.take()
+	}
+}
+
+/// This is called by the dynamically generated trampoline taking the function index and reference
+/// to the call arguments on the stack as arguments. Returns zero on success and a non-zero value
+/// on failure.
+unsafe extern "C" fn stub_fn(vmctx: *mut VMContext, func_index: u32, values_vec: *mut i64) -> u32 {
+	if let Some(state) = (*vmctx).host_state().downcast_mut::<EnvState>() {
+			match stub_fn_inner(
+				vmctx,
+				&state.host_functions,
+				&mut state.compiler,
+				state.executor_state.as_mut(),
+				func_index,
+				values_vec,
+			) {
+				Ok(()) => CALL_SUCCESS,
+				Err(err) => {
+					state.trap = Some(err);
+					CALL_FAILED_WITH_ERROR
+				}
+			}
+	} else {
+		// Well, we can't even set a trap message, so we'll just exit without one.
+		CALL_WITH_BAD_HOST_STATE
+	}
+}
+
+/// Implements most of the logic in `stub_fn` but returning a `Result` instead of an integer error
+/// for the sake of readability.
+unsafe fn stub_fn_inner(
+	vmctx: *mut VMContext,
+	externals: &[&dyn Function],
+	compiler: &mut Compiler,
+	executor_state: Option<&mut FunctionExecutorState>,
+	func_index: u32,
+	values_vec: *mut i64,
+) -> Result<(), Error> {
+	let func = externals.get(func_index as usize)
+		.ok_or_else(|| format!("call to undefined external function with index {}", func_index))?;
+	let executor_state = executor_state
+		.ok_or_else(|| "executor state is None during call to external function")?;
+
+	// Build the external function context.
+	let mut context = FunctionExecutor::new(vmctx, compiler, executor_state)?;
+	let mut context = AssertUnwindSafe(&mut context);
+
+	// Execute and write output back to the stack.
+	let return_val = panic::catch_unwind(move || {
+		let signature = func.signature();
+
+		// Read the arguments from the stack.
+		let mut args = signature.args.iter()
+			.enumerate()
+			.map(|(i, &param_type)| read_value_from(values_vec.offset(i as isize), param_type));
+
+		func.execute(&mut **context, &mut args)
+	});
+
+	match return_val {
+		Ok(ret_val) => {
+			if let Some(val) = ret_val
+				.map_err(|e| Error::FunctionExecution(func.name().to_string(), e))? {
+				write_value_to(values_vec, val);
+			}
+
+			Ok(())
+		},
+		Err(e) => {
+			let message = if let Some(err) = e.downcast_ref::<String>() {
+				err.to_string()
+			} else if let Some(err) = e.downcast_ref::<&str>() {
+				err.to_string()
+			} else {
+				"Panicked without any further information!".into()
+			};
+
+			Err(Error::FunctionExecution(func.name().to_string(), message))
+		}
+	}
+}
+
+/// Create a trampoline for invoking a host function.
+///
+/// The trampoline is a dynamically generated entry point to a runtime host call. The function is
+/// generated by manually constructing Cranelift IR and using the Cranelift compiler. The
+/// trampoline embeds the function index as a constant and delegates to a stub function in Rust,
+/// which takes the function index and a memory reference to the stack arguments and return value
+/// slots.
+///
+/// This code is of modified copy of wasmtime's wasmtime-api/src/trampoline/func.rs.
+pub fn make_trampoline(
+	isa: &dyn isa::TargetIsa,
+	code_memory: &mut CodeMemory,
+	fn_builder_ctx: &mut FunctionBuilderContext,
+	func_index: u32,
+	signature: &ir::Signature,
+) -> Result<*const VMFunctionBody, WasmError> {
+	// Mostly reverse copy of the similar method from wasmtime's
+	// wasmtime-jit/src/compiler.rs.
+	let pointer_type = isa.pointer_type();
+	let mut stub_sig = ir::Signature::new(isa.frontend_config().default_call_conv);
+
+	// Ensure that the first parameter of the generated function is the `VMContext` pointer.
+	assert_eq!(
+		signature.params[0],
+		ir::AbiParam::special(pointer_type, ir::ArgumentPurpose::VMContext)
+	);
+
+	// Add the `vmctx` parameter.
+	stub_sig.params.push(ir::AbiParam::special(
+		pointer_type,
+		ir::ArgumentPurpose::VMContext,
+	));
+
+	// Add the `func_index` parameter.
+	stub_sig.params.push(ir::AbiParam::new(ir::types::I32));
+
+	// Add the `values_vec` parameter.
+	stub_sig.params.push(ir::AbiParam::new(pointer_type));
+
+	// Add error/trap return.
+	stub_sig.returns.push(ir::AbiParam::new(ir::types::I32));
+
+	// Each parameter and return value gets a 64-bit (8-byte) wide slot on the stack, as that is
+	// large enough to fit all Wasm primitive types that can be used in host function signatures.
+	// The `VMContext` pointer, which is a parameter of the function signature, is excluded as it
+	// is passed directly to the stub function rather than being looked up on the caller stack from
+	// the `values_vec` pointer.
+	let values_vec_len = cmp::max(signature.params.len() - 1, signature.returns.len());
+	let values_vec_size = (MAX_WASM_TYPE_SIZE * values_vec_len) as u32;
+
+	let mut context = Context::new();
+	context.func =
+		ir::Function::with_name_signature(ir::ExternalName::user(0, 0), signature.clone());
+
+	let ss = context.func.create_stack_slot(StackSlotData::new(
+		StackSlotKind::ExplicitSlot,
+		values_vec_size,
+	));
+
+	{
+		let mut builder = FunctionBuilder::new(&mut context.func, fn_builder_ctx);
+		let block0 = builder.create_ebb();
+
+		builder.append_ebb_params_for_function_params(block0);
+		builder.switch_to_block(block0);
+		builder.seal_block(block0);
+
+		let values_vec_ptr_val = builder.ins().stack_addr(pointer_type, ss, 0);
+		let mflags = ir::MemFlags::trusted();
+		for i in 1..signature.params.len() {
+			let val = builder.func.dfg.ebb_params(block0)[i];
+			builder.ins().store(
+				mflags,
+				val,
+				values_vec_ptr_val,
+				((i - 1) * MAX_WASM_TYPE_SIZE) as i32,
+			);
+		}
+
+		let vmctx_ptr_val = builder.func.dfg.ebb_params(block0)[0];
+		let func_index_val = builder.ins().iconst(ir::types::I32, func_index as i64);
+
+		let callee_args = vec![vmctx_ptr_val, func_index_val, values_vec_ptr_val];
+
+		let new_sig = builder.import_signature(stub_sig.clone());
+
+		let callee_value = builder
+			.ins()
+			.iconst(pointer_type, stub_fn as *const VMFunctionBody as i64);
+		let call = builder
+			.ins()
+			.call_indirect(new_sig, callee_value, &callee_args);
+
+		let call_result = builder.func.dfg.inst_results(call)[0];
+		builder.ins().trapnz(call_result, TrapCode::User(TRAP_USER_CODE));
+
+		let mflags = ir::MemFlags::trusted();
+		let mut results = Vec::new();
+		for (i, r) in signature.returns.iter().enumerate() {
+			let load = builder.ins().load(
+				r.value_type,
+				mflags,
+				values_vec_ptr_val,
+				(i * MAX_WASM_TYPE_SIZE) as i32,
+			);
+			results.push(load);
+		}
+		builder.ins().return_(&results);
+		builder.finalize()
+	}
+
+	let mut code_buf: Vec<u8> = Vec::new();
+	let mut reloc_sink = RelocSink;
+	let mut trap_sink = binemit::NullTrapSink {};
+	let mut stackmap_sink = binemit::NullStackmapSink {};
+	context
+		.compile_and_emit(
+			isa,
+			&mut code_buf,
+			&mut reloc_sink,
+			&mut trap_sink,
+			&mut stackmap_sink,
+		)
+		.map_err(|e| WasmError::Instantiation(format!("failed to compile trampoline: {}", e)))?;
+
+	let func_ref = code_memory
+		.allocate_copy_of_byte_slice(&code_buf)
+		.map_err(|e| WasmError::Instantiation(format!("failed to allocate code memory: {}", e)))?;
+
+	Ok(func_ref.as_ptr())
+}
+
+/// We don't expect trampoline compilation to produce any relocations, so
+/// this `RelocSink` just asserts that it doesn't recieve any.
+struct RelocSink;
+
+impl binemit::RelocSink for RelocSink {
+	fn reloc_ebb(
+		&mut self,
+		_offset: binemit::CodeOffset,
+		_reloc: binemit::Reloc,
+		_ebb_offset: binemit::CodeOffset,
+	) {
+		panic!("trampoline compilation should not produce ebb relocs");
+	}
+	fn reloc_external(
+		&mut self,
+		_offset: binemit::CodeOffset,
+		_reloc: binemit::Reloc,
+		_name: &ir::ExternalName,
+		_addend: binemit::Addend,
+	) {
+		panic!("trampoline compilation should not produce external symbol relocs");
+	}
+	fn reloc_constant(
+		&mut self,
+		_code_offset: binemit::CodeOffset,
+		_reloc: binemit::Reloc,
+		_constant_offset: ir::ConstantOffset,
+	) {
+		panic!("trampoline compilation should not produce constant relocs");
+	}
+	fn reloc_jt(
+		&mut self,
+		_offset: binemit::CodeOffset,
+		_reloc: binemit::Reloc,
+		_jt: ir::JumpTable,
+	) {
+		panic!("trampoline compilation should not produce jump table relocs");
+	}
+}
+
+unsafe fn write_value_to(p: *mut i64, val: Value) {
+	match val {
+		Value::I32(i) => ptr::write(p as *mut i32, i),
+		Value::I64(i) => ptr::write(p as *mut i64, i),
+		Value::F32(u) => ptr::write(p as *mut u32, u),
+		Value::F64(u) => ptr::write(p as *mut u64, u),
+	}
+}
+
+unsafe fn read_value_from(p: *const i64, ty: ValueType) -> Value {
+	match ty {
+		ValueType::I32 => Value::I32(ptr::read(p as *const i32)),
+		ValueType::I64 => Value::I64(ptr::read(p as *const i64)),
+		ValueType::F32 => Value::F32(ptr::read(p as *const u32)),
+		ValueType::F64 => Value::F64(ptr::read(p as *const u64)),
+	}
+}

substrate/client/executor/src/wasmtime/util.rs

@@ -0,0 +1,113 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::error::{Error, Result};
+
+use cranelift_codegen::{ir, isa};
+use std::ops::Range;
+use wasm_interface::{Pointer, Signature, ValueType};
+
+/// Read data from a slice of memory into a destination buffer.
+///
+/// Returns an error if the read would go out of the memory bounds.
+pub fn read_memory_into(memory: &[u8], address: Pointer<u8>, dest: &mut [u8]) -> Result<()> {
+	let range = checked_range(address.into(), dest.len(), memory.len())
+		.ok_or_else(|| Error::Other("memory read is out of bounds".into()))?;
+	dest.copy_from_slice(&memory[range]);
+	Ok(())
+}
+
+/// Write data to a slice of memory.
+///
+/// Returns an error if the write would go out of the memory bounds.
+pub fn write_memory_from(memory: &mut [u8], address: Pointer<u8>, data: &[u8]) -> Result<()> {
+	let range = checked_range(address.into(), data.len(), memory.len())
+		.ok_or_else(|| Error::Other("memory write is out of bounds".into()))?;
+	&mut memory[range].copy_from_slice(data);
+	Ok(())
+}
+
+/// Construct a range from an offset to a data length after the offset.
+/// Returns None if the end of the range would exceed some maximum offset.
+pub fn checked_range(offset: usize, len: usize, max: usize) -> Option<Range<usize>> {
+	let end = offset.checked_add(len)?;
+	if end <= max {
+		Some(offset..end)
+	} else {
+		None
+	}
+}
+
+/// Convert a wasm_interface Signature into a cranelift_codegen Signature.
+pub fn cranelift_ir_signature(signature: Signature, call_conv: &isa::CallConv) -> ir::Signature {
+	ir::Signature {
+		params: signature.args.iter()
+			.map(cranelift_ir_type)
+			.map(ir::AbiParam::new)
+			.collect(),
+		returns: signature.return_value.iter()
+			.map(cranelift_ir_type)
+			.map(ir::AbiParam::new)
+			.collect(),
+		call_conv: call_conv.clone(),
+	}
+}
+
+/// Convert a wasm_interface ValueType into a cranelift_codegen Type.
+pub fn cranelift_ir_type(value_type: &ValueType) -> ir::types::Type {
+	match value_type {
+		ValueType::I32 => ir::types::I32,
+		ValueType::I64 => ir::types::I64,
+		ValueType::F32 => ir::types::F32,
+		ValueType::F64 => ir::types::F64,
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use assert_matches::assert_matches;
+
+	#[test]
+	fn test_read_memory_into() {
+		let mut memory = [0; 20];
+		let mut dest = [0; 5];
+
+		&mut memory[15..20].copy_from_slice(b"hello");
+
+		read_memory_into(&memory[..], Pointer::new(15), &mut dest[..]).unwrap();
+
+		// Test that out of bounds read fails.
+		assert_matches!(
+			read_memory_into(&memory[..], Pointer::new(16), &mut dest[..]),
+			Err(Error::Other(_))
+		)
+	}
+
+	#[test]
+	fn test_write_memory_from() {
+		let mut memory = [0; 20];
+		let data = b"hello";
+
+		write_memory_from(&mut memory[..], Pointer::new(15), data).unwrap();
+
+		// Test that out of bounds write fails.
+		assert_matches!(
+			write_memory_from(&mut memory[..], Pointer::new(16), data),
+			Err(Error::Other(_))
+		)
+	}
+}

substrate/client/finality-grandpa/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "substrate-finality-grandpa"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+fork-tree = { path = "../../utils/fork-tree" }
+futures = "0.1.29"
+futures03 = { package = "futures-preview", version = "0.3.0-alpha.19", features = ["compat"] }
+log = "0.4.8"
+parking_lot = "0.9.0"
+tokio-executor = "0.1.8"
+tokio-timer = "0.2.11"
+rand = "0.7.2"
+codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] }
+sr-primitives = {  path = "../../primitives/sr-primitives" }
+consensus_common = { package = "substrate-consensus-common", path = "../../primitives/consensus/common" }
+primitives = { package = "substrate-primitives",  path = "../../primitives/core" }
+substrate-telemetry = { path = "../telemetry" }
+keystore = { package = "substrate-keystore", path = "../keystore" }
+serde_json = "1.0.41"
+client-api = { package = "substrate-client-api", path = "../api" }
+client = { package = "substrate-client", path = "../" }
+header-metadata = { package = "substrate-header-metadata", path = "../header-metadata" }
+inherents = { package = "substrate-inherents", path = "../../primitives/inherents" }
+network = { package = "substrate-network", path = "../network" }
+paint-finality-tracker = { path = "../../paint/finality-tracker" }
+fg_primitives = { package = "substrate-finality-grandpa-primitives", path = "../../primitives/finality-grandpa" }
+grandpa = { package = "finality-grandpa", version = "0.9.0", features = ["derive-codec"] }
+
+[dev-dependencies]
+grandpa = { package = "finality-grandpa", version = "0.9.0", features = ["derive-codec", "test-helpers"] }
+network = { package = "substrate-network", path = "../network", features = ["test-helpers"] }
+keyring = { package = "substrate-keyring", path = "../../primitives/keyring" }
+test-client = { package = "substrate-test-runtime-client", path = "../../test/utils/runtime/client"}
+babe_primitives = { package = "substrate-consensus-babe-primitives", path = "../../primitives/consensus/babe" }
+state_machine = { package = "substrate-state-machine", path = "../../primitives/state-machine" }
+env_logger = "0.7.0"
+tokio = "0.1.22"
+tempfile = "3.1.0"
+sr-api = { path = "../../primitives/sr-api" }

substrate/client/finality-grandpa/src/authorities.rs

@@ -0,0 +1,787 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Utilities for dealing with authorities, authority sets, and handoffs.
+
+use fork_tree::ForkTree;
+use parking_lot::RwLock;
+use grandpa::voter_set::VoterSet;
+use codec::{Encode, Decode};
+use log::{debug, info};
+use substrate_telemetry::{telemetry, CONSENSUS_INFO};
+use fg_primitives::{AuthorityId, AuthorityList};
+
+use std::cmp::Ord;
+use std::fmt::Debug;
+use std::ops::Add;
+use std::sync::Arc;
+
+/// A shared authority set.
+pub(crate) struct SharedAuthoritySet<H, N> {
+	inner: Arc<RwLock<AuthoritySet<H, N>>>,
+}
+
+impl<H, N> Clone for SharedAuthoritySet<H, N> {
+	fn clone(&self) -> Self {
+		SharedAuthoritySet { inner: self.inner.clone() }
+	}
+}
+
+impl<H, N> SharedAuthoritySet<H, N> {
+	/// Acquire a reference to the inner read-write lock.
+	pub(crate) fn inner(&self) -> &RwLock<AuthoritySet<H, N>> {
+		&*self.inner
+	}
+}
+
+impl<H: Eq, N> SharedAuthoritySet<H, N>
+where N: Add<Output=N> + Ord + Clone + Debug,
+	  H: Clone + Debug
+{
+	/// Get the earliest limit-block number, if any.
+	pub(crate) fn current_limit(&self) -> Option<N> {
+		self.inner.read().current_limit()
+	}
+
+	/// Get the current set ID. This is incremented every time the set changes.
+	pub(crate) fn set_id(&self) -> u64 {
+		self.inner.read().set_id
+	}
+
+	/// Get the current authorities and their weights (for the current set ID).
+	pub(crate) fn current_authorities(&self) -> VoterSet<AuthorityId> {
+		self.inner.read().current_authorities.iter().cloned().collect()
+	}
+}
+
+impl<H, N> From<AuthoritySet<H, N>> for SharedAuthoritySet<H, N> {
+	fn from(set: AuthoritySet<H, N>) -> Self {
+		SharedAuthoritySet { inner: Arc::new(RwLock::new(set)) }
+	}
+}
+
+/// Status of the set after changes were applied.
+#[derive(Debug)]
+pub(crate) struct Status<H, N> {
+	/// Whether internal changes were made.
+	pub(crate) changed: bool,
+	/// `Some` when underlying authority set has changed, containing the
+	/// block where that set changed.
+	pub(crate) new_set_block: Option<(H, N)>,
+}
+
+/// A set of authorities.
+#[derive(Debug, Clone, Encode, Decode, PartialEq)]
+pub(crate) struct AuthoritySet<H, N> {
+	pub(crate) current_authorities: AuthorityList,
+	pub(crate) set_id: u64,
+	// Tree of pending standard changes across forks. Standard changes are
+	// enacted on finality and must be enacted (i.e. finalized) in-order across
+	// a given branch
+	pub(crate) pending_standard_changes: ForkTree<H, N, PendingChange<H, N>>,
+	// Pending forced changes across different forks (at most one per fork).
+	// Forced changes are enacted on block depth (not finality), for this reason
+	// only one forced change should exist per fork.
+	pub(crate) pending_forced_changes: Vec<PendingChange<H, N>>,
+}
+
+impl<H, N> AuthoritySet<H, N>
+where H: PartialEq,
+	  N: Ord,
+{
+	/// Get a genesis set with given authorities.
+	pub(crate) fn genesis(initial: AuthorityList) -> Self {
+		AuthoritySet {
+			current_authorities: initial,
+			set_id: 0,
+			pending_standard_changes: ForkTree::new(),
+			pending_forced_changes: Vec::new(),
+		}
+	}
+
+	/// Get the current set id and a reference to the current authority set.
+	pub(crate) fn current(&self) -> (u64, &[(AuthorityId, u64)]) {
+		(self.set_id, &self.current_authorities[..])
+	}
+}
+
+impl<H: Eq, N> AuthoritySet<H, N>
+where
+	N: Add<Output=N> + Ord + Clone + Debug,
+	H: Clone + Debug
+{
+	fn add_standard_change<F, E>(
+		&mut self,
+		pending: PendingChange<H, N>,
+		is_descendent_of: &F,
+	) -> Result<(), fork_tree::Error<E>> where
+		F: Fn(&H, &H) -> Result<bool, E>,
+		E:  std::error::Error,
+	{
+		let hash = pending.canon_hash.clone();
+		let number = pending.canon_height.clone();
+
+		debug!(target: "afg", "Inserting potential standard set change signaled at block {:?} \
+							   (delayed by {:?} blocks).",
+			   (&number, &hash), pending.delay);
+
+		self.pending_standard_changes.import(
+			hash.clone(),
+			number.clone(),
+			pending,
+			is_descendent_of,
+		)?;
+
+		debug!(target: "afg", "There are now {} alternatives for the next pending standard change (roots), \
+							   and a total of {} pending standard changes (across all forks).",
+			self.pending_standard_changes.roots().count(),
+			self.pending_standard_changes.iter().count(),
+		);
+
+		Ok(())
+	}
+
+	fn add_forced_change<F, E>(
+		&mut self,
+		pending: PendingChange<H, N>,
+		is_descendent_of: &F,
+	) -> Result<(), fork_tree::Error<E>> where
+		F: Fn(&H, &H) -> Result<bool, E>,
+		E:  std::error::Error,
+	{
+		for change in self.pending_forced_changes.iter() {
+			if change.canon_hash == pending.canon_hash ||
+				is_descendent_of(&change.canon_hash, &pending.canon_hash)?
+			{
+				return Err(fork_tree::Error::UnfinalizedAncestor);
+			}
+		}
+
+		// ordered first by effective number and then by signal-block number.
+		let key = (pending.effective_number(), pending.canon_height.clone());
+		let idx = self.pending_forced_changes
+			.binary_search_by_key(&key, |change| (
+				change.effective_number(),
+				change.canon_height.clone(),
+			))
+			.unwrap_or_else(|i| i);
+
+		debug!(target: "afg", "Inserting potential forced set change at block {:?} \
+							   (delayed by {:?} blocks).",
+			   (&pending.canon_height, &pending.canon_hash), pending.delay);
+
+		self.pending_forced_changes.insert(idx, pending);
+
+		debug!(target: "afg", "There are now {} pending forced changes.", self.pending_forced_changes.len());
+
+		Ok(())
+	}
+
+	/// Note an upcoming pending transition. Multiple pending standard changes
+	/// on the same branch can be added as long as they don't overlap. Forced
+	/// changes are restricted to one per fork. This method assumes that changes
+	/// on the same branch will be added in-order. The given function
+	/// `is_descendent_of` should return `true` if the second hash (target) is a
+	/// descendent of the first hash (base).
+	pub(crate) fn add_pending_change<F, E>(
+		&mut self,
+		pending: PendingChange<H, N>,
+		is_descendent_of: &F,
+	) -> Result<(), fork_tree::Error<E>> where
+		F: Fn(&H, &H) -> Result<bool, E>,
+		E:  std::error::Error,
+	{
+		match pending.delay_kind {
+			DelayKind::Best { .. } => {
+				self.add_forced_change(pending, is_descendent_of)
+			},
+			DelayKind::Finalized => {
+				self.add_standard_change(pending, is_descendent_of)
+			},
+		}
+	}
+
+	/// Inspect pending changes. Standard pending changes are iterated first,
+	/// and the changes in the tree are traversed in pre-order, afterwards all
+	/// forced changes are iterated.
+	pub(crate) fn pending_changes(&self) -> impl Iterator<Item=&PendingChange<H, N>> {
+		self.pending_standard_changes.iter().map(|(_, _, c)| c)
+			.chain(self.pending_forced_changes.iter())
+	}
+
+	/// Get the earliest limit-block number, if any. If there are pending changes across
+	/// different forks, this method will return the earliest effective number (across the
+	/// different branches). Only standard changes are taken into account for the current
+	/// limit, since any existing forced change should preclude the voter from voting.
+	pub(crate) fn current_limit(&self) -> Option<N> {
+		self.pending_standard_changes.roots()
+			.min_by_key(|&(_, _, c)| c.effective_number())
+			.map(|(_, _, c)| c.effective_number())
+	}
+
+	/// Apply or prune any pending transitions based on a best-block trigger.
+	///
+	/// Returns `Ok((median, new_set))` when a forced change has occurred. The
+	/// median represents the median last finalized block at the time the change
+	/// was signaled, and it should be used as the canon block when starting the
+	/// new grandpa voter. Only alters the internal state in this case.
+	///
+	/// These transitions are always forced and do not lead to justifications
+	/// which light clients can follow.
+	pub(crate) fn apply_forced_changes<F, E>(
+		&self,
+		best_hash: H,
+		best_number: N,
+		is_descendent_of: &F,
+	) -> Result<Option<(N, Self)>, E>
+		where F: Fn(&H, &H) -> Result<bool, E>,
+	{
+		let mut new_set = None;
+
+		for change in self.pending_forced_changes.iter()
+			.take_while(|c| c.effective_number() <= best_number) // to prevent iterating too far
+			.filter(|c| c.effective_number() == best_number)
+		{
+			// check if the given best block is in the same branch as the block that signaled the change.
+			if is_descendent_of(&change.canon_hash, &best_hash)? {
+				// apply this change: make the set canonical
+				info!(target: "finality", "Applying authority set change forced at block #{:?}",
+					  change.canon_height);
+				telemetry!(CONSENSUS_INFO; "afg.applying_forced_authority_set_change";
+					"block" => ?change.canon_height
+				);
+
+				let median_last_finalized = match change.delay_kind {
+					DelayKind::Best { ref median_last_finalized } => median_last_finalized.clone(),
+					_ => unreachable!("pending_forced_changes only contains forced changes; forced changes have delay kind Best; qed."),
+				};
+
+				new_set = Some((median_last_finalized, AuthoritySet {
+					current_authorities: change.next_authorities.clone(),
+					set_id: self.set_id + 1,
+					pending_standard_changes: ForkTree::new(), // new set, new changes.
+					pending_forced_changes: Vec::new(),
+				}));
+
+				break;
+			}
+
+			// we don't wipe forced changes until another change is
+			// applied
+		}
+
+		Ok(new_set)
+	}
+
+	/// Apply or prune any pending transitions based on a finality trigger. This
+	/// method ensures that if there are multiple changes in the same branch,
+	/// finalizing this block won't finalize past multiple transitions (i.e.
+	/// transitions must be finalized in-order). The given function
+	/// `is_descendent_of` should return `true` if the second hash (target) is a
+	/// descendent of the first hash (base).
+	///
+	/// When the set has changed, the return value will be `Ok(Some((H, N)))`
+	/// which is the canonical block where the set last changed (i.e. the given
+	/// hash and number).
+	pub(crate) fn apply_standard_changes<F, E>(
+		&mut self,
+		finalized_hash: H,
+		finalized_number: N,
+		is_descendent_of: &F,
+	) -> Result<Status<H, N>, fork_tree::Error<E>>
+		where F: Fn(&H, &H) -> Result<bool, E>,
+			  E: std::error::Error,
+	{
+		let mut status = Status {
+			changed: false,
+			new_set_block: None,
+		};
+
+		match self.pending_standard_changes.finalize_with_descendent_if(
+			&finalized_hash,
+			finalized_number.clone(),
+			is_descendent_of,
+			|change| change.effective_number() <= finalized_number
+		)? {
+			fork_tree::FinalizationResult::Changed(change) => {
+				status.changed = true;
+
+				// if we are able to finalize any standard change then we can
+				// discard all pending forced changes (on different forks)
+				self.pending_forced_changes.clear();
+
+				if let Some(change) = change {
+					info!(target: "finality", "Applying authority set change scheduled at block #{:?}",
+						  change.canon_height);
+					telemetry!(CONSENSUS_INFO; "afg.applying_scheduled_authority_set_change";
+						"block" => ?change.canon_height
+					);
+
+					self.current_authorities = change.next_authorities;
+					self.set_id += 1;
+
+					status.new_set_block = Some((
+						finalized_hash,
+						finalized_number,
+					));
+				}
+			},
+			fork_tree::FinalizationResult::Unchanged => {},
+		}
+
+		Ok(status)
+	}
+
+	/// Check whether the given finalized block number enacts any standard
+	/// authority set change (without triggering it), ensuring that if there are
+	/// multiple changes in the same branch, finalizing this block won't
+	/// finalize past multiple transitions (i.e. transitions must be finalized
+	/// in-order). Returns `Some(true)` if the block being finalized enacts a
+	/// change that can be immediately applied, `Some(false)` if the block being
+	/// finalized enacts a change but it cannot be applied yet since there are
+	/// other dependent changes, and `None` if no change is enacted. The given
+	/// function `is_descendent_of` should return `true` if the second hash
+	/// (target) is a descendent of the first hash (base).
+	pub fn enacts_standard_change<F, E>(
+		&self,
+		finalized_hash: H,
+		finalized_number: N,
+		is_descendent_of: &F,
+	) -> Result<Option<bool>, fork_tree::Error<E>>
+	where F: Fn(&H, &H) -> Result<bool, E>,
+		  E: std::error::Error,
+	{
+		self.pending_standard_changes.finalizes_any_with_descendent_if(
+			&finalized_hash,
+			finalized_number.clone(),
+			is_descendent_of,
+			|change| change.effective_number() == finalized_number
+		)
+	}
+}
+
+/// Kinds of delays for pending changes.
+#[derive(Debug, Clone, Encode, Decode, PartialEq)]
+pub(crate) enum DelayKind<N> {
+	/// Depth in finalized chain.
+	Finalized,
+	/// Depth in best chain. The median last finalized block is calculated at the time the
+	/// change was signaled.
+	Best { median_last_finalized: N },
+}
+
+/// A pending change to the authority set.
+///
+/// This will be applied when the announcing block is at some depth within
+/// the finalized or unfinalized chain.
+#[derive(Debug, Clone, Encode, PartialEq)]
+pub(crate) struct PendingChange<H, N> {
+	/// The new authorities and weights to apply.
+	pub(crate) next_authorities: AuthorityList,
+	/// How deep in the chain the announcing block must be
+	/// before the change is applied.
+	pub(crate) delay: N,
+	/// The announcing block's height.
+	pub(crate) canon_height: N,
+	/// The announcing block's hash.
+	pub(crate) canon_hash: H,
+	/// The delay kind.
+	pub(crate) delay_kind: DelayKind<N>,
+}
+
+impl<H: Decode, N: Decode> Decode for PendingChange<H, N> {
+	fn decode<I: codec::Input>(value: &mut I) -> Result<Self, codec::Error> {
+		let next_authorities = Decode::decode(value)?;
+		let delay = Decode::decode(value)?;
+		let canon_height = Decode::decode(value)?;
+		let canon_hash = Decode::decode(value)?;
+
+		let delay_kind = DelayKind::decode(value).unwrap_or(DelayKind::Finalized);
+
+		Ok(PendingChange {
+			next_authorities,
+			delay,
+			canon_height,
+			canon_hash,
+			delay_kind,
+		})
+	}
+}
+
+impl<H, N: Add<Output=N> + Clone> PendingChange<H, N> {
+	/// Returns the effective number this change will be applied at.
+	pub fn effective_number(&self) -> N {
+		self.canon_height.clone() + self.delay.clone()
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use primitives::crypto::Public;
+
+	fn static_is_descendent_of<A>(value: bool)
+		-> impl Fn(&A, &A) -> Result<bool, std::io::Error>
+	{
+		move |_, _| Ok(value)
+	}
+
+	fn is_descendent_of<A, F>(f: F) -> impl Fn(&A, &A) -> Result<bool, std::io::Error>
+		where F: Fn(&A, &A) -> bool
+	{
+		move |base, hash| Ok(f(base, hash))
+	}
+
+	#[test]
+	fn changes_iterated_in_pre_order() {
+		let mut authorities = AuthoritySet {
+			current_authorities: Vec::new(),
+			set_id: 0,
+			pending_standard_changes: ForkTree::new(),
+			pending_forced_changes: Vec::new(),
+		};
+
+		let change_a = PendingChange {
+			next_authorities: Vec::new(),
+			delay: 10,
+			canon_height: 5,
+			canon_hash: "hash_a",
+			delay_kind: DelayKind::Finalized,
+		};
+
+		let change_b = PendingChange {
+			next_authorities: Vec::new(),
+			delay: 0,
+			canon_height: 5,
+			canon_hash: "hash_b",
+			delay_kind: DelayKind::Finalized,
+		};
+
+		let change_c = PendingChange {
+			next_authorities: Vec::new(),
+			delay: 5,
+			canon_height: 10,
+			canon_hash: "hash_c",
+			delay_kind: DelayKind::Finalized,
+		};
+
+		authorities.add_pending_change(change_a.clone(), &static_is_descendent_of(false)).unwrap();
+		authorities.add_pending_change(change_b.clone(), &static_is_descendent_of(false)).unwrap();
+		authorities.add_pending_change(change_c.clone(), &is_descendent_of(|base, hash| match (*base, *hash) {
+			("hash_a", "hash_c") => true,
+			("hash_b", "hash_c") => false,
+			_ => unreachable!(),
+		})).unwrap();
+
+		// forced changes are iterated last
+		let change_d = PendingChange {
+			next_authorities: Vec::new(),
+			delay: 2,
+			canon_height: 1,
+			canon_hash: "hash_d",
+			delay_kind: DelayKind::Best { median_last_finalized: 0 },
+		};
+
+		let change_e = PendingChange {
+			next_authorities: Vec::new(),
+			delay: 2,
+			canon_height: 0,
+			canon_hash: "hash_e",
+			delay_kind: DelayKind::Best { median_last_finalized: 0 },
+		};
+
+		authorities.add_pending_change(change_d.clone(), &static_is_descendent_of(false)).unwrap();
+		authorities.add_pending_change(change_e.clone(), &static_is_descendent_of(false)).unwrap();
+
+		assert_eq!(
+			authorities.pending_changes().collect::<Vec<_>>(),
+			vec![&change_b, &change_a, &change_c, &change_e, &change_d],
+		);
+	}
+
+	#[test]
+	fn apply_change() {
+		let mut authorities = AuthoritySet {
+			current_authorities: Vec::new(),
+			set_id: 0,
+			pending_standard_changes: ForkTree::new(),
+			pending_forced_changes: Vec::new(),
+		};
+
+		let set_a = vec![(AuthorityId::from_slice(&[1; 32]), 5)];
+		let set_b = vec![(AuthorityId::from_slice(&[2; 32]), 5)];
+
+		// two competing changes at the same height on different forks
+		let change_a = PendingChange {
+			next_authorities: set_a.clone(),
+			delay: 10,
+			canon_height: 5,
+			canon_hash: "hash_a",
+			delay_kind: DelayKind::Finalized,
+		};
+
+		let change_b = PendingChange {
+			next_authorities: set_b.clone(),
+			delay: 10,
+			canon_height: 5,
+			canon_hash: "hash_b",
+			delay_kind: DelayKind::Finalized,
+		};
+
+		authorities.add_pending_change(change_a.clone(), &static_is_descendent_of(true)).unwrap();
+		authorities.add_pending_change(change_b.clone(), &static_is_descendent_of(true)).unwrap();
+
+		assert_eq!(
+			authorities.pending_changes().collect::<Vec<_>>(),
+			vec![&change_b, &change_a],
+		);
+
+		// finalizing "hash_c" won't enact the change signaled at "hash_a" but it will prune out "hash_b"
+		let status = authorities.apply_standard_changes("hash_c", 11, &is_descendent_of(|base, hash| match (*base, *hash) {
+			("hash_a", "hash_c") => true,
+			("hash_b", "hash_c") => false,
+			_ => unreachable!(),
+		})).unwrap();
+
+		assert!(status.changed);
+		assert_eq!(status.new_set_block, None);
+		assert_eq!(
+			authorities.pending_changes().collect::<Vec<_>>(),
+			vec![&change_a],
+		);
+
+		// finalizing "hash_d" will enact the change signaled at "hash_a"
+		let status = authorities.apply_standard_changes("hash_d", 15, &is_descendent_of(|base, hash| match (*base, *hash) {
+			("hash_a", "hash_d") => true,
+			_ => unreachable!(),
+		})).unwrap();
+
+		assert!(status.changed);
+		assert_eq!(status.new_set_block, Some(("hash_d", 15)));
+
+		assert_eq!(authorities.current_authorities, set_a);
+		assert_eq!(authorities.set_id, 1);
+		assert_eq!(authorities.pending_changes().count(), 0);
+	}
+
+	#[test]
+	fn disallow_multiple_changes_being_finalized_at_once() {
+		let mut authorities = AuthoritySet {
+			current_authorities: Vec::new(),
+			set_id: 0,
+			pending_standard_changes: ForkTree::new(),
+			pending_forced_changes: Vec::new(),
+		};
+
+		let set_a = vec![(AuthorityId::from_slice(&[1; 32]), 5)];
+		let set_c = vec![(AuthorityId::from_slice(&[2; 32]), 5)];
+
+		// two competing changes at the same height on different forks
+		let change_a = PendingChange {
+			next_authorities: set_a.clone(),
+			delay: 10,
+			canon_height: 5,
+			canon_hash: "hash_a",
+			delay_kind: DelayKind::Finalized,
+		};
+
+		let change_c = PendingChange {
+			next_authorities: set_c.clone(),
+			delay: 10,
+			canon_height: 30,
+			canon_hash: "hash_c",
+			delay_kind: DelayKind::Finalized,
+		};
+
+		authorities.add_pending_change(change_a.clone(), &static_is_descendent_of(true)).unwrap();
+		authorities.add_pending_change(change_c.clone(), &static_is_descendent_of(true)).unwrap();
+
+		let is_descendent_of = is_descendent_of(|base, hash| match (*base, *hash) {
+			("hash_a", "hash_b") => true,
+			("hash_a", "hash_c") => true,
+			("hash_a", "hash_d") => true,
+
+			("hash_c", "hash_b") => false,
+			("hash_c", "hash_d") => true,
+
+			("hash_b", "hash_c") => true,
+			_ => unreachable!(),
+		});
+
+		// trying to finalize past `change_c` without finalizing `change_a` first
+		match authorities.apply_standard_changes("hash_d", 40, &is_descendent_of) {
+			Err(fork_tree::Error::UnfinalizedAncestor) => {},
+			_ => unreachable!(),
+		}
+
+		let status = authorities.apply_standard_changes("hash_b", 15, &is_descendent_of).unwrap();
+		assert!(status.changed);
+		assert_eq!(status.new_set_block, Some(("hash_b", 15)));
+
+		assert_eq!(authorities.current_authorities, set_a);
+		assert_eq!(authorities.set_id, 1);
+
+		// after finalizing `change_a` it should be possible to finalize `change_c`
+		let status = authorities.apply_standard_changes("hash_d", 40, &is_descendent_of).unwrap();
+		assert!(status.changed);
+		assert_eq!(status.new_set_block, Some(("hash_d", 40)));
+
+		assert_eq!(authorities.current_authorities, set_c);
+		assert_eq!(authorities.set_id, 2);
+	}
+
+	#[test]
+	fn enacts_standard_change_works() {
+		let mut authorities = AuthoritySet {
+			current_authorities: Vec::new(),
+			set_id: 0,
+			pending_standard_changes: ForkTree::new(),
+			pending_forced_changes: Vec::new(),
+		};
+
+		let set_a = vec![(AuthorityId::from_slice(&[1; 32]), 5)];
+
+		let change_a = PendingChange {
+			next_authorities: set_a.clone(),
+			delay: 10,
+			canon_height: 5,
+			canon_hash: "hash_a",
+			delay_kind: DelayKind::Finalized,
+		};
+
+		let change_b = PendingChange {
+			next_authorities: set_a.clone(),
+			delay: 10,
+			canon_height: 20,
+			canon_hash: "hash_b",
+			delay_kind: DelayKind::Finalized,
+		};
+
+		authorities.add_pending_change(change_a.clone(), &static_is_descendent_of(false)).unwrap();
+		authorities.add_pending_change(change_b.clone(), &static_is_descendent_of(true)).unwrap();
+
+		let is_descendent_of = is_descendent_of(|base, hash| match (*base, *hash) {
+			("hash_a", "hash_d") => true,
+			("hash_a", "hash_e") => true,
+			("hash_b", "hash_d") => true,
+			("hash_b", "hash_e") => true,
+			("hash_a", "hash_c") => false,
+			("hash_b", "hash_c") => false,
+			_ => unreachable!(),
+		});
+
+		// "hash_c" won't finalize the existing change since it isn't a descendent
+		assert_eq!(
+			authorities.enacts_standard_change("hash_c", 15, &is_descendent_of).unwrap(),
+			None,
+		);
+
+		// "hash_d" at depth 14 won't work either
+		assert_eq!(
+			authorities.enacts_standard_change("hash_d", 14, &is_descendent_of).unwrap(),
+			None,
+		);
+
+		// but it should work at depth 15 (change height + depth)
+		assert_eq!(
+			authorities.enacts_standard_change("hash_d", 15, &is_descendent_of).unwrap(),
+			Some(true),
+		);
+
+		// finalizing "hash_e" at depth 20 will trigger change at "hash_b", but
+		// it can't be applied yet since "hash_a" must be applied first
+		assert_eq!(
+			authorities.enacts_standard_change("hash_e", 30, &is_descendent_of).unwrap(),
+			Some(false),
+		);
+	}
+
+	#[test]
+	fn forced_changes() {
+		let mut authorities = AuthoritySet {
+			current_authorities: Vec::new(),
+			set_id: 0,
+			pending_standard_changes: ForkTree::new(),
+			pending_forced_changes: Vec::new(),
+		};
+
+		let set_a = vec![(AuthorityId::from_slice(&[1; 32]), 5)];
+		let set_b = vec![(AuthorityId::from_slice(&[2; 32]), 5)];
+
+		let change_a = PendingChange {
+			next_authorities: set_a.clone(),
+			delay: 10,
+			canon_height: 5,
+			canon_hash: "hash_a",
+			delay_kind: DelayKind::Best { median_last_finalized: 42 },
+		};
+
+		let change_b = PendingChange {
+			next_authorities: set_b.clone(),
+			delay: 10,
+			canon_height: 5,
+			canon_hash: "hash_b",
+			delay_kind: DelayKind::Best { median_last_finalized: 0 },
+		};
+
+		authorities.add_pending_change(change_a, &static_is_descendent_of(false)).unwrap();
+		authorities.add_pending_change(change_b, &static_is_descendent_of(false)).unwrap();
+
+		// there's an effective change triggered at block 15 but not a standard one.
+		// so this should do nothing.
+		assert_eq!(
+			authorities.enacts_standard_change("hash_c", 15, &static_is_descendent_of(true)).unwrap(),
+			None,
+		);
+
+		// throw a standard change into the mix to prove that it's discarded
+		// for being on the same fork.
+		//
+		// NOTE: after https://github.com/paritytech/substrate/issues/1861
+		// this should still be rejected based on the "span" rule -- it overlaps
+		// with another change on the same fork.
+		let change_c = PendingChange {
+			next_authorities: set_b.clone(),
+			delay: 3,
+			canon_height: 8,
+			canon_hash: "hash_a8",
+			delay_kind: DelayKind::Best { median_last_finalized: 0 },
+		};
+
+		let is_descendent_of_a = is_descendent_of(|base: &&str, _| {
+			base.starts_with("hash_a")
+		});
+
+		assert!(authorities.add_pending_change(change_c, &is_descendent_of_a).is_err());
+
+		// too early.
+		assert!(authorities.apply_forced_changes("hash_a10", 10, &static_is_descendent_of(true)).unwrap().is_none());
+
+		// too late.
+		assert!(authorities.apply_forced_changes("hash_a16", 16, &static_is_descendent_of(true)).unwrap().is_none());
+
+		// on time -- chooses the right change.
+		assert_eq!(
+			authorities.apply_forced_changes("hash_a15", 15, &is_descendent_of_a).unwrap().unwrap(),
+			(42, AuthoritySet {
+				current_authorities: set_a,
+				set_id: 1,
+				pending_standard_changes: ForkTree::new(),
+				pending_forced_changes: Vec::new(),
+			})
+		);
+	}
+}

substrate/client/finality-grandpa/src/aux_schema.rs

@@ -0,0 +1,611 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Schema for stuff in the aux-db.
+
+use std::fmt::Debug;
+use std::sync::Arc;
+use codec::{Encode, Decode};
+use client_api::backend::AuxStore;
+use client_api::error::{Result as ClientResult, Error as ClientError};
+use fork_tree::ForkTree;
+use grandpa::round::State as RoundState;
+use sr_primitives::traits::{Block as BlockT, NumberFor};
+use log::{info, warn};
+use fg_primitives::{AuthorityList, SetId, RoundNumber};
+
+use crate::authorities::{AuthoritySet, SharedAuthoritySet, PendingChange, DelayKind};
+use crate::consensus_changes::{SharedConsensusChanges, ConsensusChanges};
+use crate::environment::{
+	CompletedRound, CompletedRounds, CurrentRounds, HasVoted, SharedVoterSetState, VoterSetState,
+};
+use crate::NewAuthoritySet;
+
+const VERSION_KEY: &[u8] = b"grandpa_schema_version";
+const SET_STATE_KEY: &[u8] = b"grandpa_completed_round";
+const AUTHORITY_SET_KEY: &[u8] = b"grandpa_voters";
+const CONSENSUS_CHANGES_KEY: &[u8] = b"grandpa_consensus_changes";
+
+const CURRENT_VERSION: u32 = 2;
+
+/// The voter set state.
+#[derive(Debug, Clone, Encode, Decode)]
+#[cfg_attr(test, derive(PartialEq))]
+pub enum V1VoterSetState<H, N> {
+	/// The voter set state, currently paused.
+	Paused(RoundNumber, RoundState<H, N>),
+	/// The voter set state, currently live.
+	Live(RoundNumber, RoundState<H, N>),
+}
+
+type V0VoterSetState<H, N> = (RoundNumber, RoundState<H, N>);
+
+#[derive(Debug, Clone, Encode, Decode, PartialEq)]
+struct V0PendingChange<H, N> {
+	next_authorities: AuthorityList,
+	delay: N,
+	canon_height: N,
+	canon_hash: H,
+}
+
+#[derive(Debug, Clone, Encode, Decode, PartialEq)]
+struct V0AuthoritySet<H, N> {
+	current_authorities: AuthorityList,
+	set_id: SetId,
+	pending_changes: Vec<V0PendingChange<H, N>>,
+}
+
+impl<H, N> Into<AuthoritySet<H, N>> for V0AuthoritySet<H, N>
+where H: Clone + Debug + PartialEq,
+	  N: Clone + Debug + Ord,
+{
+	fn into(self) -> AuthoritySet<H, N> {
+		let mut pending_standard_changes = ForkTree::new();
+
+		for old_change in self.pending_changes {
+			let new_change = PendingChange {
+				next_authorities: old_change.next_authorities,
+				delay: old_change.delay,
+				canon_height: old_change.canon_height,
+				canon_hash: old_change.canon_hash,
+				delay_kind: DelayKind::Finalized,
+			};
+
+			if let Err(err) = pending_standard_changes.import::<_, ClientError>(
+				new_change.canon_hash.clone(),
+				new_change.canon_height.clone(),
+				new_change,
+				// previously we only supported at most one pending change per fork
+				&|_, _| Ok(false),
+			) {
+				warn!(target: "afg", "Error migrating pending authority set change: {:?}.", err);
+				warn!(target: "afg", "Node is in a potentially inconsistent state.");
+			}
+		}
+
+		AuthoritySet {
+			current_authorities: self.current_authorities,
+			set_id: self.set_id,
+			pending_forced_changes: Vec::new(),
+			pending_standard_changes
+		}
+	}
+}
+
+pub(crate) fn load_decode<B: AuxStore, T: Decode>(backend: &B, key: &[u8]) -> ClientResult<Option<T>> {
+	match backend.get_aux(key)? {
+		None => Ok(None),
+		Some(t) => T::decode(&mut &t[..])
+			.map_err(
+				|e| ClientError::Backend(format!("GRANDPA DB is corrupted: {}", e.what())),
+			)
+			.map(Some)
+	}
+}
+
+/// Persistent data kept between runs.
+pub(crate) struct PersistentData<Block: BlockT> {
+	pub(crate) authority_set: SharedAuthoritySet<Block::Hash, NumberFor<Block>>,
+	pub(crate) consensus_changes: SharedConsensusChanges<Block::Hash, NumberFor<Block>>,
+	pub(crate) set_state: SharedVoterSetState<Block>,
+}
+
+fn migrate_from_version0<Block: BlockT, B, G>(
+	backend: &B,
+	genesis_round: &G,
+) -> ClientResult<Option<(
+	AuthoritySet<Block::Hash, NumberFor<Block>>,
+	VoterSetState<Block>,
+)>> where B: AuxStore,
+		  G: Fn() -> RoundState<Block::Hash, NumberFor<Block>>,
+{
+	CURRENT_VERSION.using_encoded(|s|
+		backend.insert_aux(&[(VERSION_KEY, s)], &[])
+	)?;
+
+	if let Some(old_set) = load_decode::<_, V0AuthoritySet<Block::Hash, NumberFor<Block>>>(
+		backend,
+		AUTHORITY_SET_KEY,
+	)? {
+		let new_set: AuthoritySet<Block::Hash, NumberFor<Block>> = old_set.into();
+		backend.insert_aux(&[(AUTHORITY_SET_KEY, new_set.encode().as_slice())], &[])?;
+
+		let (last_round_number, last_round_state) = match load_decode::<_, V0VoterSetState<Block::Hash, NumberFor<Block>>>(
+			backend,
+			SET_STATE_KEY,
+		)? {
+			Some((number, state)) => (number, state),
+			None => (0, genesis_round()),
+		};
+
+		let set_id = new_set.current().0;
+
+		let base = last_round_state.prevote_ghost
+			.expect("state is for completed round; completed rounds must have a prevote ghost; qed.");
+
+		let mut current_rounds = CurrentRounds::new();
+		current_rounds.insert(last_round_number + 1, HasVoted::No);
+
+		let set_state = VoterSetState::Live {
+			completed_rounds: CompletedRounds::new(
+				CompletedRound {
+					number: last_round_number,
+					state: last_round_state,
+					votes: Vec::new(),
+					base,
+				},
+				set_id,
+				&new_set,
+			),
+			current_rounds,
+		};
+
+		backend.insert_aux(&[(SET_STATE_KEY, set_state.encode().as_slice())], &[])?;
+
+		return Ok(Some((new_set, set_state)));
+	}
+
+	Ok(None)
+}
+
+fn migrate_from_version1<Block: BlockT, B, G>(
+	backend: &B,
+	genesis_round: &G,
+) -> ClientResult<Option<(
+	AuthoritySet<Block::Hash, NumberFor<Block>>,
+	VoterSetState<Block>,
+)>> where B: AuxStore,
+		  G: Fn() -> RoundState<Block::Hash, NumberFor<Block>>,
+{
+	CURRENT_VERSION.using_encoded(|s|
+		backend.insert_aux(&[(VERSION_KEY, s)], &[])
+	)?;
+
+	if let Some(set) = load_decode::<_, AuthoritySet<Block::Hash, NumberFor<Block>>>(
+		backend,
+		AUTHORITY_SET_KEY,
+	)? {
+		let set_id = set.current().0;
+
+		let completed_rounds = |number, state, base| CompletedRounds::new(
+			CompletedRound {
+				number,
+				state,
+				votes: Vec::new(),
+				base,
+			},
+			set_id,
+			&set,
+		);
+
+		let set_state = match load_decode::<_, V1VoterSetState<Block::Hash, NumberFor<Block>>>(
+			backend,
+			SET_STATE_KEY,
+		)? {
+			Some(V1VoterSetState::Paused(last_round_number, set_state)) => {
+				let base = set_state.prevote_ghost
+					.expect("state is for completed round; completed rounds must have a prevote ghost; qed.");
+
+				VoterSetState::Paused {
+					completed_rounds: completed_rounds(last_round_number, set_state, base),
+				}
+			},
+			Some(V1VoterSetState::Live(last_round_number, set_state)) => {
+				let base = set_state.prevote_ghost
+					.expect("state is for completed round; completed rounds must have a prevote ghost; qed.");
+
+				let mut current_rounds = CurrentRounds::new();
+				current_rounds.insert(last_round_number + 1, HasVoted::No);
+
+				VoterSetState::Live {
+					completed_rounds: completed_rounds(last_round_number, set_state, base),
+					current_rounds,
+				}
+			},
+			None => {
+				let set_state = genesis_round();
+				let base = set_state.prevote_ghost
+					.expect("state is for completed round; completed rounds must have a prevote ghost; qed.");
+
+				VoterSetState::live(
+					set_id,
+					&set,
+					base,
+				)
+			},
+		};
+
+		backend.insert_aux(&[(SET_STATE_KEY, set_state.encode().as_slice())], &[])?;
+
+		return Ok(Some((set, set_state)));
+	}
+
+	Ok(None)
+}
+
+/// Load or initialize persistent data from backend.
+pub(crate) fn load_persistent<Block: BlockT, B, G>(
+	backend: &B,
+	genesis_hash: Block::Hash,
+	genesis_number: NumberFor<Block>,
+	genesis_authorities: G,
+)
+	-> ClientResult<PersistentData<Block>>
+	where
+		B: AuxStore,
+		G: FnOnce() -> ClientResult<AuthorityList>,
+{
+	let version: Option<u32> = load_decode(backend, VERSION_KEY)?;
+	let consensus_changes = load_decode(backend, CONSENSUS_CHANGES_KEY)?
+		.unwrap_or_else(ConsensusChanges::<Block::Hash, NumberFor<Block>>::empty);
+
+	let make_genesis_round = move || RoundState::genesis((genesis_hash, genesis_number));
+
+	match version {
+		None => {
+			if let Some((new_set, set_state)) = migrate_from_version0::<Block, _, _>(backend, &make_genesis_round)? {
+				return Ok(PersistentData {
+					authority_set: new_set.into(),
+					consensus_changes: Arc::new(consensus_changes.into()),
+					set_state: set_state.into(),
+				});
+			}
+		},
+		Some(1) => {
+			if let Some((new_set, set_state)) = migrate_from_version1::<Block, _, _>(backend, &make_genesis_round)? {
+				return Ok(PersistentData {
+					authority_set: new_set.into(),
+					consensus_changes: Arc::new(consensus_changes.into()),
+					set_state: set_state.into(),
+				});
+			}
+		},
+		Some(2) => {
+			if let Some(set) = load_decode::<_, AuthoritySet<Block::Hash, NumberFor<Block>>>(
+				backend,
+				AUTHORITY_SET_KEY,
+			)? {
+				let set_state = match load_decode::<_, VoterSetState<Block>>(
+					backend,
+					SET_STATE_KEY,
+				)? {
+					Some(state) => state,
+					None => {
+						let state = make_genesis_round();
+						let base = state.prevote_ghost
+							.expect("state is for completed round; completed rounds must have a prevote ghost; qed.");
+
+						VoterSetState::live(
+							set.current().0,
+							&set,
+							base,
+						)
+					}
+				};
+
+				return Ok(PersistentData {
+					authority_set: set.into(),
+					consensus_changes: Arc::new(consensus_changes.into()),
+					set_state: set_state.into(),
+				});
+			}
+		}
+		Some(other) => return Err(ClientError::Backend(
+			format!("Unsupported GRANDPA DB version: {:?}", other)
+		).into()),
+	}
+
+	// genesis.
+	info!(target: "afg", "Loading GRANDPA authority set \
+		from genesis on what appears to be first startup.");
+
+	let genesis_authorities = genesis_authorities()?;
+	let genesis_set = AuthoritySet::genesis(genesis_authorities.clone());
+	let state = make_genesis_round();
+	let base = state.prevote_ghost
+		.expect("state is for completed round; completed rounds must have a prevote ghost; qed.");
+
+	let genesis_state = VoterSetState::live(
+		0,
+		&genesis_set,
+		base,
+	);
+
+	backend.insert_aux(
+		&[
+			(AUTHORITY_SET_KEY, genesis_set.encode().as_slice()),
+			(SET_STATE_KEY, genesis_state.encode().as_slice()),
+		],
+		&[],
+	)?;
+
+	Ok(PersistentData {
+		authority_set: genesis_set.into(),
+		set_state: genesis_state.into(),
+		consensus_changes: Arc::new(consensus_changes.into()),
+	})
+}
+
+/// Update the authority set on disk after a change.
+///
+/// If there has just been a handoff, pass a `new_set` parameter that describes the
+/// handoff. `set` in all cases should reflect the current authority set, with all
+/// changes and handoffs applied.
+pub(crate) fn update_authority_set<Block: BlockT, F, R>(
+	set: &AuthoritySet<Block::Hash, NumberFor<Block>>,
+	new_set: Option<&NewAuthoritySet<Block::Hash, NumberFor<Block>>>,
+	write_aux: F
+) -> R where
+	F: FnOnce(&[(&'static [u8], &[u8])]) -> R,
+{
+	// write new authority set state to disk.
+	let encoded_set = set.encode();
+
+	if let Some(new_set) = new_set {
+		// we also overwrite the "last completed round" entry with a blank slate
+		// because from the perspective of the finality gadget, the chain has
+		// reset.
+		let set_state = VoterSetState::<Block>::live(
+			new_set.set_id,
+			&set,
+			(new_set.canon_hash, new_set.canon_number),
+		);
+		let encoded = set_state.encode();
+
+		write_aux(&[
+			(AUTHORITY_SET_KEY, &encoded_set[..]),
+			(SET_STATE_KEY, &encoded[..]),
+		])
+	} else {
+		write_aux(&[(AUTHORITY_SET_KEY, &encoded_set[..])])
+	}
+}
+
+/// Write voter set state.
+pub(crate) fn write_voter_set_state<Block: BlockT, B: AuxStore>(
+	backend: &B,
+	state: &VoterSetState<Block>,
+) -> ClientResult<()> {
+	backend.insert_aux(
+		&[(SET_STATE_KEY, state.encode().as_slice())],
+		&[]
+	)
+}
+
+/// Update the consensus changes.
+pub(crate) fn update_consensus_changes<H, N, F, R>(
+	set: &ConsensusChanges<H, N>,
+	write_aux: F
+) -> R where
+	H: Encode + Clone,
+	N: Encode + Clone,
+	F: FnOnce(&[(&'static [u8], &[u8])]) -> R,
+{
+	write_aux(&[(CONSENSUS_CHANGES_KEY, set.encode().as_slice())])
+}
+
+#[cfg(test)]
+pub(crate) fn load_authorities<B: AuxStore, H: Decode, N: Decode>(backend: &B)
+	-> Option<AuthoritySet<H, N>> {
+	load_decode::<_, AuthoritySet<H, N>>(backend, AUTHORITY_SET_KEY)
+		.expect("backend error")
+}
+
+#[cfg(test)]
+mod test {
+	use fg_primitives::AuthorityId;
+	use primitives::H256;
+	use test_client;
+	use super::*;
+
+	#[test]
+	fn load_decode_from_v0_migrates_data_format() {
+		let client = test_client::new();
+
+		let authorities = vec![(AuthorityId::default(), 100)];
+		let set_id = 3;
+		let round_number: RoundNumber = 42;
+		let round_state = RoundState::<H256, u64> {
+			prevote_ghost: Some((H256::random(), 32)),
+			finalized: None,
+			estimate: None,
+			completable: false,
+		};
+
+		{
+			let authority_set = V0AuthoritySet::<H256, u64> {
+				current_authorities: authorities.clone(),
+				pending_changes: Vec::new(),
+				set_id,
+			};
+
+			let voter_set_state = (round_number, round_state.clone());
+
+			client.insert_aux(
+				&[
+					(AUTHORITY_SET_KEY, authority_set.encode().as_slice()),
+					(SET_STATE_KEY, voter_set_state.encode().as_slice()),
+				],
+				&[],
+			).unwrap();
+		}
+
+		assert_eq!(
+			load_decode::<_, u32>(&client, VERSION_KEY).unwrap(),
+			None,
+		);
+
+		// should perform the migration
+		load_persistent::<test_client::runtime::Block, _, _>(
+			&client,
+			H256::random(),
+			0,
+			|| unreachable!(),
+		).unwrap();
+
+		assert_eq!(
+			load_decode::<_, u32>(&client, VERSION_KEY).unwrap(),
+			Some(2),
+		);
+
+		let PersistentData { authority_set, set_state, .. } = load_persistent::<test_client::runtime::Block, _, _>(
+			&client,
+			H256::random(),
+			0,
+			|| unreachable!(),
+		).unwrap();
+
+		assert_eq!(
+			*authority_set.inner().read(),
+			AuthoritySet {
+				current_authorities: authorities.clone(),
+				pending_standard_changes: ForkTree::new(),
+				pending_forced_changes: Vec::new(),
+				set_id,
+			},
+		);
+
+		let mut current_rounds = CurrentRounds::new();
+		current_rounds.insert(round_number + 1, HasVoted::No);
+
+		assert_eq!(
+			&*set_state.read(),
+			&VoterSetState::Live {
+				completed_rounds: CompletedRounds::new(
+					CompletedRound {
+						number: round_number,
+						state: round_state.clone(),
+						base: round_state.prevote_ghost.unwrap(),
+						votes: vec![],
+					},
+					set_id,
+					&*authority_set.inner().read(),
+				),
+				current_rounds,
+			},
+		);
+	}
+
+	#[test]
+	fn load_decode_from_v1_migrates_data_format() {
+		let client = test_client::new();
+
+		let authorities = vec![(AuthorityId::default(), 100)];
+		let set_id = 3;
+		let round_number: RoundNumber = 42;
+		let round_state = RoundState::<H256, u64> {
+			prevote_ghost: Some((H256::random(), 32)),
+			finalized: None,
+			estimate: None,
+			completable: false,
+		};
+
+		{
+			let authority_set = AuthoritySet::<H256, u64> {
+				current_authorities: authorities.clone(),
+				pending_standard_changes: ForkTree::new(),
+				pending_forced_changes: Vec::new(),
+				set_id,
+			};
+
+			let voter_set_state = V1VoterSetState::Live(round_number, round_state.clone());
+
+			client.insert_aux(
+				&[
+					(AUTHORITY_SET_KEY, authority_set.encode().as_slice()),
+					(SET_STATE_KEY, voter_set_state.encode().as_slice()),
+					(VERSION_KEY, 1u32.encode().as_slice()),
+				],
+				&[],
+			).unwrap();
+		}
+
+		assert_eq!(
+			load_decode::<_, u32>(&client, VERSION_KEY).unwrap(),
+			Some(1),
+		);
+
+		// should perform the migration
+		load_persistent::<test_client::runtime::Block, _, _>(
+			&client,
+			H256::random(),
+			0,
+			|| unreachable!(),
+		).unwrap();
+
+		assert_eq!(
+			load_decode::<_, u32>(&client, VERSION_KEY).unwrap(),
+			Some(2),
+		);
+
+		let PersistentData { authority_set, set_state, .. } = load_persistent::<test_client::runtime::Block, _, _>(
+			&client,
+			H256::random(),
+			0,
+			|| unreachable!(),
+		).unwrap();
+
+		assert_eq!(
+			*authority_set.inner().read(),
+			AuthoritySet {
+				current_authorities: authorities.clone(),
+				pending_standard_changes: ForkTree::new(),
+				pending_forced_changes: Vec::new(),
+				set_id,
+			},
+		);
+
+		let mut current_rounds = CurrentRounds::new();
+		current_rounds.insert(round_number + 1, HasVoted::No);
+
+		assert_eq!(
+			&*set_state.read(),
+			&VoterSetState::Live {
+				completed_rounds: CompletedRounds::new(
+					CompletedRound {
+						number: round_number,
+						state: round_state.clone(),
+						base: round_state.prevote_ghost.unwrap(),
+						votes: vec![],
+					},
+					set_id,
+					&*authority_set.inner().read(),
+				),
+				current_rounds,
+			},
+		);
+	}
+}

substrate/client/finality-grandpa/src/communication/periodic.rs

@@ -0,0 +1,110 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Periodic rebroadcast of neighbor packets.
+
+use std::time::{Instant, Duration};
+
+use codec::Encode;
+use futures::prelude::*;
+use futures::sync::mpsc;
+use log::{debug, warn};
+use tokio_timer::Delay;
+
+use network::PeerId;
+use sr_primitives::traits::{NumberFor, Block as BlockT};
+use super::{gossip::{NeighborPacket, GossipMessage}, Network};
+
+// how often to rebroadcast, if no other
+const REBROADCAST_AFTER: Duration = Duration::from_secs(2 * 60);
+
+fn rebroadcast_instant() -> Instant {
+	Instant::now() + REBROADCAST_AFTER
+}
+
+/// A sender used to send neighbor packets to a background job.
+#[derive(Clone)]
+pub(super) struct NeighborPacketSender<B: BlockT>(
+	mpsc::UnboundedSender<(Vec<PeerId>, NeighborPacket<NumberFor<B>>)>
+);
+
+impl<B: BlockT> NeighborPacketSender<B> {
+	/// Send a neighbor packet for the background worker to gossip to peers.
+	pub fn send(
+		&self,
+		who: Vec<network::PeerId>,
+		neighbor_packet: NeighborPacket<NumberFor<B>>,
+	) {
+		if let Err(err) = self.0.unbounded_send((who, neighbor_packet)) {
+			debug!(target: "afg", "Failed to send neighbor packet: {:?}", err);
+		}
+	}
+}
+
+/// Does the work of sending neighbor packets, asynchronously.
+///
+/// It may rebroadcast the last neighbor packet periodically when no
+/// progress is made.
+pub(super) fn neighbor_packet_worker<B, N>(net: N) -> (
+	impl Future<Item = (), Error = ()> + Send + 'static,
+	NeighborPacketSender<B>,
+) where
+	B: BlockT,
+	N: Network<B>,
+{
+	let mut last = None;
+	let (tx, mut rx) = mpsc::unbounded::<(Vec<PeerId>, NeighborPacket<NumberFor<B>>)>();
+	let mut delay = Delay::new(rebroadcast_instant());
+
+	let work = futures::future::poll_fn(move || {
+		loop {
+			match rx.poll().expect("unbounded receivers do not error; qed") {
+				Async::Ready(None) => return Ok(Async::Ready(())),
+				Async::Ready(Some((to, packet))) => {
+					// send to peers.
+					net.send_message(to.clone(), GossipMessage::<B>::from(packet.clone()).encode());
+
+					// rebroadcasting network.
+					delay.reset(rebroadcast_instant());
+					last = Some((to, packet));
+				}
+				Async::NotReady => break,
+			}
+		}
+
+		// has to be done in a loop because it needs to be polled after
+		// re-scheduling.
+		loop {
+			match delay.poll() {
+				Err(e) => {
+					warn!(target: "afg", "Could not rebroadcast neighbor packets: {:?}", e);
+					delay.reset(rebroadcast_instant());
+				}
+				Ok(Async::Ready(())) => {
+					delay.reset(rebroadcast_instant());
+
+					if let Some((ref to, ref packet)) = last {
+						// send to peers.
+						net.send_message(to.clone(), GossipMessage::<B>::from(packet.clone()).encode());
+					}
+				}
+				Ok(Async::NotReady) => return Ok(Async::NotReady),
+			}
+		}
+	});
+
+	(work, NeighborPacketSender(tx))
+}

substrate/client/finality-grandpa/src/communication/tests.rs

@@ -0,0 +1,506 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Tests for the communication portion of the GRANDPA crate.
+
+use futures::sync::mpsc;
+use futures::prelude::*;
+use network::consensus_gossip as network_gossip;
+use network::test::{Block, Hash};
+use network_gossip::Validator;
+use tokio::runtime::current_thread;
+use std::sync::Arc;
+use keyring::Ed25519Keyring;
+use codec::Encode;
+use sr_primitives::traits::NumberFor;
+
+use crate::environment::SharedVoterSetState;
+use fg_primitives::AuthorityList;
+use super::gossip::{self, GossipValidator};
+use super::{AuthorityId, VoterSet, Round, SetId};
+
+enum Event {
+	MessagesFor(Hash, mpsc::UnboundedSender<network_gossip::TopicNotification>),
+	RegisterValidator(Arc<dyn network_gossip::Validator<Block>>),
+	GossipMessage(Hash, Vec<u8>, bool),
+	SendMessage(Vec<network::PeerId>, Vec<u8>),
+	Report(network::PeerId, i32),
+	Announce(Hash),
+}
+
+#[derive(Clone)]
+struct TestNetwork {
+	sender: mpsc::UnboundedSender<Event>,
+}
+
+impl super::Network<Block> for TestNetwork {
+	type In = mpsc::UnboundedReceiver<network_gossip::TopicNotification>;
+
+	/// Get a stream of messages for a specific gossip topic.
+	fn messages_for(&self, topic: Hash) -> Self::In {
+		let (tx, rx) = mpsc::unbounded();
+		let _ = self.sender.unbounded_send(Event::MessagesFor(topic, tx));
+
+		rx
+	}
+
+	/// Register a gossip validator.
+	fn register_validator(&self, validator: Arc<dyn network_gossip::Validator<Block>>) {
+		let _ = self.sender.unbounded_send(Event::RegisterValidator(validator));
+	}
+
+	/// Gossip a message out to all connected peers.
+	///
+	/// Force causes it to be sent to all peers, even if they've seen it already.
+	/// Only should be used in case of consensus stall.
+	fn gossip_message(&self, topic: Hash, data: Vec<u8>, force: bool) {
+		let _ = self.sender.unbounded_send(Event::GossipMessage(topic, data, force));
+	}
+
+	/// Send a message to a bunch of specific peers, even if they've seen it already.
+	fn send_message(&self, who: Vec<network::PeerId>, data: Vec<u8>) {
+		let _ = self.sender.unbounded_send(Event::SendMessage(who, data));
+	}
+
+	/// Register a message with the gossip service, it isn't broadcast right
+	/// away to any peers, but may be sent to new peers joining or when asked to
+	/// broadcast the topic. Useful to register previous messages on node
+	/// startup.
+	fn register_gossip_message(&self, _topic: Hash, _data: Vec<u8>) {
+		// NOTE: only required to restore previous state on startup
+		//       not required for tests currently
+	}
+
+	/// Report a peer's cost or benefit after some action.
+	fn report(&self, who: network::PeerId, cost_benefit: i32) {
+		let _ = self.sender.unbounded_send(Event::Report(who, cost_benefit));
+	}
+
+	/// Inform peers that a block with given hash should be downloaded.
+	fn announce(&self, block: Hash, _associated_data: Vec<u8>) {
+		let _ = self.sender.unbounded_send(Event::Announce(block));
+	}
+
+	/// Notify the sync service to try syncing the given chain.
+	fn set_sync_fork_request(&self, _peers: Vec<network::PeerId>, _hash: Hash, _number: NumberFor<Block>) {}
+}
+
+impl network_gossip::ValidatorContext<Block> for TestNetwork {
+	fn broadcast_topic(&mut self, _: Hash, _: bool) { }
+
+	fn broadcast_message(&mut self, _: Hash, _: Vec<u8>, _: bool) {	}
+
+	fn send_message(&mut self, who: &network::PeerId, data: Vec<u8>) {
+		<Self as super::Network<Block>>::send_message(self, vec![who.clone()], data);
+	}
+
+	fn send_topic(&mut self, _: &network::PeerId, _: Hash, _: bool) { }
+}
+
+struct Tester {
+	net_handle: super::NetworkBridge<Block, TestNetwork>,
+	gossip_validator: Arc<GossipValidator<Block>>,
+	events: mpsc::UnboundedReceiver<Event>,
+}
+
+impl Tester {
+	fn filter_network_events<F>(self, mut pred: F) -> impl Future<Item=Self,Error=()>
+		where F: FnMut(Event) -> bool
+	{
+		let mut s = Some(self);
+		futures::future::poll_fn(move || loop {
+			match s.as_mut().unwrap().events.poll().expect("concluded early") {
+				Async::Ready(None) => panic!("concluded early"),
+				Async::Ready(Some(item)) => if pred(item) {
+					return Ok(Async::Ready(s.take().unwrap()))
+				},
+				Async::NotReady => return Ok(Async::NotReady),
+			}
+		})
+	}
+}
+
+// some random config (not really needed)
+fn config() -> crate::Config {
+	crate::Config {
+		gossip_duration: std::time::Duration::from_millis(10),
+		justification_period: 256,
+		keystore: None,
+		name: None,
+		is_authority: true,
+		observer_enabled: true,
+	}
+}
+
+// dummy voter set state
+fn voter_set_state() -> SharedVoterSetState<Block> {
+	use crate::authorities::AuthoritySet;
+	use crate::environment::VoterSetState;
+	use grandpa::round::State as RoundState;
+	use primitives::H256;
+
+	let state = RoundState::genesis((H256::zero(), 0));
+	let base = state.prevote_ghost.unwrap();
+	let voters = AuthoritySet::genesis(Vec::new());
+	let set_state = VoterSetState::live(
+		0,
+		&voters,
+		base,
+	);
+
+	set_state.into()
+}
+
+// needs to run in a tokio runtime.
+fn make_test_network() -> (
+	impl Future<Item=Tester,Error=()>,
+	TestNetwork,
+) {
+	let (tx, rx) = mpsc::unbounded();
+	let net = TestNetwork { sender: tx };
+
+	#[derive(Clone)]
+	struct Exit;
+
+	impl Future for Exit {
+		type Item = ();
+		type Error = ();
+
+		fn poll(&mut self) -> Poll<(), ()> {
+			Ok(Async::NotReady)
+		}
+	}
+
+	let (bridge, startup_work) = super::NetworkBridge::new(
+		net.clone(),
+		config(),
+		voter_set_state(),
+		Exit,
+	);
+
+	(
+		startup_work.map(move |()| Tester {
+			gossip_validator: bridge.validator.clone(),
+			net_handle: bridge,
+			events: rx,
+		}),
+		net,
+	)
+}
+
+fn make_ids(keys: &[Ed25519Keyring]) -> AuthorityList {
+	keys.iter()
+		.map(|key| key.clone().public().into())
+		.map(|id| (id, 1))
+		.collect()
+}
+
+struct NoopContext;
+
+impl network_gossip::ValidatorContext<Block> for NoopContext {
+	fn broadcast_topic(&mut self, _: Hash, _: bool) { }
+	fn broadcast_message(&mut self, _: Hash, _: Vec<u8>, _: bool) { }
+	fn send_message(&mut self, _: &network::PeerId, _: Vec<u8>) { }
+	fn send_topic(&mut self, _: &network::PeerId, _: Hash, _: bool) { }
+}
+
+#[test]
+fn good_commit_leads_to_relay() {
+	let private = [Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie];
+	let public = make_ids(&private[..]);
+	let voter_set = Arc::new(public.iter().cloned().collect::<VoterSet<AuthorityId>>());
+
+	let round = 1;
+	let set_id = 1;
+
+	let commit = {
+		let target_hash: Hash = [1; 32].into();
+		let target_number = 500;
+
+		let precommit = grandpa::Precommit { target_hash: target_hash.clone(), target_number };
+		let payload = super::localized_payload(
+			round, set_id, &grandpa::Message::Precommit(precommit.clone())
+		);
+
+		let mut precommits = Vec::new();
+		let mut auth_data = Vec::new();
+
+		for (i, key) in private.iter().enumerate() {
+			precommits.push(precommit.clone());
+
+			let signature = fg_primitives::AuthoritySignature::from(key.sign(&payload[..]));
+			auth_data.push((signature, public[i].0.clone()))
+		}
+
+		grandpa::CompactCommit {
+			target_hash,
+			target_number,
+			precommits,
+			auth_data,
+		}
+	};
+
+	let encoded_commit = gossip::GossipMessage::<Block>::Commit(gossip::FullCommitMessage {
+		round: Round(round),
+		set_id: SetId(set_id),
+		message: commit,
+	}).encode();
+
+	let id = network::PeerId::random();
+	let global_topic = super::global_topic::<Block>(set_id);
+
+	let test = make_test_network().0
+		.and_then(move |tester| {
+			// register a peer.
+			tester.gossip_validator.new_peer(&mut NoopContext, &id, network::config::Roles::FULL);
+			Ok((tester, id))
+		})
+		.and_then(move |(tester, id)| {
+			// start round, dispatch commit, and wait for broadcast.
+			let (commits_in, _) = tester.net_handle.global_communication(SetId(1), voter_set, false);
+
+			{
+				let (action, ..) = tester.gossip_validator.do_validate(&id, &encoded_commit[..]);
+				match action {
+					gossip::Action::ProcessAndDiscard(t, _) => assert_eq!(t, global_topic),
+					_ => panic!("wrong expected outcome from initial commit validation"),
+				}
+			}
+
+			let commit_to_send = encoded_commit.clone();
+
+			// asking for global communication will cause the test network
+			// to send us an event asking us for a stream. use it to
+			// send a message.
+			let sender_id = id.clone();
+			let send_message = tester.filter_network_events(move |event| match event {
+				Event::MessagesFor(topic, sender) => {
+					if topic != global_topic { return false }
+					let _ = sender.unbounded_send(network_gossip::TopicNotification {
+						message: commit_to_send.clone(),
+						sender: Some(sender_id.clone()),
+					});
+
+					true
+				}
+				_ => false,
+			});
+
+			// when the commit comes in, we'll tell the callback it was good.
+			let handle_commit = commits_in.into_future()
+				.map(|(item, _)| {
+					match item.unwrap() {
+						grandpa::voter::CommunicationIn::Commit(_, _, mut callback) => {
+							callback.run(grandpa::voter::CommitProcessingOutcome::good());
+						},
+						_ => panic!("commit expected"),
+					}
+				})
+				.map_err(|_| panic!("could not process commit"));
+
+			// once the message is sent and commit is "handled" we should have
+			// a repropagation event coming from the network.
+			send_message.join(handle_commit).and_then(move |(tester, ())| {
+				tester.filter_network_events(move |event| match event {
+					Event::GossipMessage(topic, data, false) => {
+						if topic == global_topic && data == encoded_commit {
+							true
+						} else {
+							panic!("Trying to gossip something strange")
+						}
+					}
+					_ => false,
+				})
+			})
+				.map_err(|_| panic!("could not watch for gossip message"))
+				.map(|_| ())
+		});
+
+	current_thread::block_on_all(test).unwrap();
+}
+
+#[test]
+fn bad_commit_leads_to_report() {
+	let private = [Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie];
+	let public = make_ids(&private[..]);
+	let voter_set = Arc::new(public.iter().cloned().collect::<VoterSet<AuthorityId>>());
+
+	let round = 1;
+	let set_id = 1;
+
+	let commit = {
+		let target_hash: Hash = [1; 32].into();
+		let target_number = 500;
+
+		let precommit = grandpa::Precommit { target_hash: target_hash.clone(), target_number };
+		let payload = super::localized_payload(
+			round, set_id, &grandpa::Message::Precommit(precommit.clone())
+		);
+
+		let mut precommits = Vec::new();
+		let mut auth_data = Vec::new();
+
+		for (i, key) in private.iter().enumerate() {
+			precommits.push(precommit.clone());
+
+			let signature = fg_primitives::AuthoritySignature::from(key.sign(&payload[..]));
+			auth_data.push((signature, public[i].0.clone()))
+		}
+
+		grandpa::CompactCommit {
+			target_hash,
+			target_number,
+			precommits,
+			auth_data,
+		}
+	};
+
+	let encoded_commit = gossip::GossipMessage::<Block>::Commit(gossip::FullCommitMessage {
+		round: Round(round),
+		set_id: SetId(set_id),
+		message: commit,
+	}).encode();
+
+	let id = network::PeerId::random();
+	let global_topic = super::global_topic::<Block>(set_id);
+
+	let test = make_test_network().0
+		.and_then(move |tester| {
+			// register a peer.
+			tester.gossip_validator.new_peer(&mut NoopContext, &id, network::config::Roles::FULL);
+			Ok((tester, id))
+		})
+		.and_then(move |(tester, id)| {
+			// start round, dispatch commit, and wait for broadcast.
+			let (commits_in, _) = tester.net_handle.global_communication(SetId(1), voter_set, false);
+
+			{
+				let (action, ..) = tester.gossip_validator.do_validate(&id, &encoded_commit[..]);
+				match action {
+					gossip::Action::ProcessAndDiscard(t, _) => assert_eq!(t, global_topic),
+					_ => panic!("wrong expected outcome from initial commit validation"),
+				}
+			}
+
+			let commit_to_send = encoded_commit.clone();
+
+			// asking for global communication will cause the test network
+			// to send us an event asking us for a stream. use it to
+			// send a message.
+			let sender_id = id.clone();
+			let send_message = tester.filter_network_events(move |event| match event {
+				Event::MessagesFor(topic, sender) => {
+					if topic != global_topic { return false }
+					let _ = sender.unbounded_send(network_gossip::TopicNotification {
+						message: commit_to_send.clone(),
+						sender: Some(sender_id.clone()),
+					});
+
+					true
+				}
+				_ => false,
+			});
+
+			// when the commit comes in, we'll tell the callback it was good.
+			let handle_commit = commits_in.into_future()
+				.map(|(item, _)| {
+					match item.unwrap() {
+						grandpa::voter::CommunicationIn::Commit(_, _, mut callback) => {
+							callback.run(grandpa::voter::CommitProcessingOutcome::bad());
+						},
+						_ => panic!("commit expected"),
+					}
+				})
+				.map_err(|_| panic!("could not process commit"));
+
+			// once the message is sent and commit is "handled" we should have
+			// a report event coming from the network.
+			send_message.join(handle_commit).and_then(move |(tester, ())| {
+				tester.filter_network_events(move |event| match event {
+					Event::Report(who, cost_benefit) => {
+						if who == id && cost_benefit == super::cost::INVALID_COMMIT {
+							true
+						} else {
+							panic!("reported unknown peer or unexpected cost");
+						}
+					}
+					_ => false,
+				})
+			})
+				.map_err(|_| panic!("could not watch for peer report"))
+				.map(|_| ())
+		});
+
+	current_thread::block_on_all(test).unwrap();
+}
+
+#[test]
+fn peer_with_higher_view_leads_to_catch_up_request() {
+	let id = network::PeerId::random();
+
+	let (tester, mut net) = make_test_network();
+	let test = tester
+		.and_then(move |tester| {
+			// register a peer with authority role.
+			tester.gossip_validator.new_peer(&mut NoopContext, &id, network::config::Roles::AUTHORITY);
+			Ok((tester, id))
+		})
+		.and_then(move |(tester, id)| {
+			// send neighbor message at round 10 and height 50
+			let result = tester.gossip_validator.validate(
+				&mut net,
+				&id,
+				&gossip::GossipMessage::<Block>::from(gossip::NeighborPacket {
+					set_id: SetId(0),
+					round: Round(10),
+					commit_finalized_height: 50,
+				}).encode(),
+			);
+
+			// neighbor packets are always discard
+			match result {
+				network_gossip::ValidationResult::Discard => {},
+				_ => panic!("wrong expected outcome from neighbor validation"),
+			}
+
+			// a catch up request should be sent to the peer for round - 1
+			tester.filter_network_events(move |event| match event {
+				Event::SendMessage(peers, message) => {
+					assert_eq!(
+						peers,
+						vec![id.clone()],
+					);
+
+					assert_eq!(
+						message,
+						gossip::GossipMessage::<Block>::CatchUpRequest(
+							gossip::CatchUpRequestMessage {
+								set_id: SetId(0),
+								round: Round(9),
+							}
+						).encode(),
+					);
+
+					true
+				},
+				_ => false,
+			})
+				.map_err(|_| panic!("could not watch for peer send message"))
+				.map(|_| ())
+		});
+
+	current_thread::block_on_all(test).unwrap();
+}

substrate/client/finality-grandpa/src/consensus_changes.rs

@@ -0,0 +1,78 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+use codec::{Encode, Decode};
+
+/// Consensus-related data changes tracker.
+#[derive(Clone, Debug, Encode, Decode)]
+pub(crate) struct ConsensusChanges<H, N> {
+	pending_changes: Vec<(N, H)>,
+}
+
+impl<H, N> ConsensusChanges<H, N> {
+	/// Create empty consensus changes.
+	pub(crate) fn empty() -> Self {
+		ConsensusChanges { pending_changes: Vec::new(), }
+	}
+}
+
+impl<H: Copy + PartialEq, N: Copy + Ord> ConsensusChanges<H, N> {
+
+	/// Returns reference to all pending changes.
+	pub fn pending_changes(&self) -> &[(N, H)] {
+		&self.pending_changes
+	}
+
+	/// Note unfinalized change of consensus-related data.
+	pub(crate) fn note_change(&mut self, at: (N, H)) {
+		let idx = self.pending_changes
+			.binary_search_by_key(&at.0, |change| change.0)
+			.unwrap_or_else(|i| i);
+		self.pending_changes.insert(idx, at);
+	}
+
+	/// Finalize all pending consensus changes that are finalized by given block.
+	/// Returns true if there any changes were finalized.
+	pub(crate) fn finalize<F: Fn(N) -> ::client_api::error::Result<Option<H>>>(
+		&mut self,
+		block: (N, H),
+		canonical_at_height: F,
+	) -> ::client_api::error::Result<(bool, bool)> {
+		let (split_idx, has_finalized_changes) = self.pending_changes.iter()
+			.enumerate()
+			.take_while(|(_, &(at_height, _))| at_height <= block.0)
+			.fold((None, Ok(false)), |(_, has_finalized_changes), (idx, ref at)|
+				(
+					Some(idx),
+					has_finalized_changes
+						.and_then(|has_finalized_changes| if has_finalized_changes {
+							Ok(has_finalized_changes)
+						} else {
+							canonical_at_height(at.0).map(|can_hash| Some(at.1) == can_hash)
+						}),
+				));
+
+		let altered_changes = split_idx.is_some();
+		if let Some(split_idx) = split_idx {
+			self.pending_changes = self.pending_changes.split_off(split_idx + 1);
+		}
+		has_finalized_changes.map(|has_finalized_changes| (altered_changes, has_finalized_changes))
+	}
+}
+
+/// Thread-safe consensus changes tracker reference.
+pub(crate) type SharedConsensusChanges<H, N> = Arc<parking_lot::Mutex<ConsensusChanges<H, N>>>;

substrate/client/finality-grandpa/src/finality_proof.rs

@@ -0,0 +1,990 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! GRANDPA block finality proof generation and check.
+//!
+//! Finality of block B is proved by providing:
+//! 1) the justification for the descendant block F;
+//! 2) headers sub-chain (B; F] if B != F;
+//! 3) proof of GRANDPA::authorities() if the set changes at block F.
+//!
+//! Since earliest possible justification is returned, the GRANDPA authorities set
+//! at the block F is guaranteed to be the same as in the block B (this is because block
+//! that enacts new GRANDPA authorities set always comes with justification). It also
+//! means that the `set_id` is the same at blocks B and F.
+//!
+//! Let U be the last finalized block known to caller. If authorities set has changed several
+//! times in the (U; F] interval, multiple finality proof fragments are returned (one for each
+//! authority set change) and they must be verified in-order.
+//!
+//! Finality proof provider can choose how to provide finality proof on its own. The incomplete
+//! finality proof (that finalizes some block C that is ancestor of the B and descendant
+//! of the U) could be returned.
+
+use std::iter;
+use std::sync::Arc;
+use log::{trace, warn};
+
+use client_api::{
+	backend::Backend, blockchain::Backend as BlockchainBackend, CallExecutor,
+	error::{Error as ClientError, Result as ClientResult},
+	light::{FetchChecker, RemoteReadRequest},
+	StorageProof,
+};
+use client::Client;
+use codec::{Encode, Decode};
+use grandpa::BlockNumberOps;
+use sr_primitives::{
+	Justification, generic::BlockId,
+	traits::{NumberFor, Block as BlockT, Header as HeaderT, One},
+};
+use primitives::{H256, Blake2Hasher, storage::StorageKey};
+use substrate_telemetry::{telemetry, CONSENSUS_INFO};
+use fg_primitives::{AuthorityId, AuthorityList, VersionedAuthorityList, GRANDPA_AUTHORITIES_KEY};
+
+use crate::justification::GrandpaJustification;
+
+/// Maximum number of fragments that we want to return in a single prove_finality call.
+const MAX_FRAGMENTS_IN_PROOF: usize = 8;
+
+/// GRANDPA authority set related methods for the finality proof provider.
+pub trait AuthoritySetForFinalityProver<Block: BlockT>: Send + Sync {
+	/// Read GRANDPA_AUTHORITIES_KEY from storage at given block.
+	fn authorities(&self, block: &BlockId<Block>) -> ClientResult<AuthorityList>;
+	/// Prove storage read of GRANDPA_AUTHORITIES_KEY at given block.
+	fn prove_authorities(&self, block: &BlockId<Block>) -> ClientResult<StorageProof>;
+}
+
+/// Client-based implementation of AuthoritySetForFinalityProver.
+impl<B, E, Block: BlockT<Hash=H256>, RA> AuthoritySetForFinalityProver<Block> for Client<B, E, Block, RA>
+	where
+		B: Backend<Block, Blake2Hasher> + Send + Sync + 'static,
+		E: CallExecutor<Block, Blake2Hasher> + 'static + Clone + Send + Sync,
+		RA: Send + Sync,
+{
+	fn authorities(&self, block: &BlockId<Block>) -> ClientResult<AuthorityList> {
+		let storage_key = StorageKey(GRANDPA_AUTHORITIES_KEY.to_vec());
+		self.storage(block, &storage_key)?
+			.and_then(|encoded| VersionedAuthorityList::decode(&mut encoded.0.as_slice()).ok())
+			.map(|versioned| versioned.into())
+			.ok_or(ClientError::InvalidAuthoritiesSet)
+	}
+
+	fn prove_authorities(&self, block: &BlockId<Block>) -> ClientResult<StorageProof> {
+		self.read_proof(block, iter::once(GRANDPA_AUTHORITIES_KEY))
+	}
+}
+
+/// GRANDPA authority set related methods for the finality proof checker.
+pub trait AuthoritySetForFinalityChecker<Block: BlockT>: Send + Sync {
+	/// Check storage read proof of GRANDPA_AUTHORITIES_KEY at given block.
+	fn check_authorities_proof(
+		&self,
+		hash: Block::Hash,
+		header: Block::Header,
+		proof: StorageProof,
+	) -> ClientResult<AuthorityList>;
+}
+
+/// FetchChecker-based implementation of AuthoritySetForFinalityChecker.
+impl<Block: BlockT> AuthoritySetForFinalityChecker<Block> for Arc<dyn FetchChecker<Block>> {
+	fn check_authorities_proof(
+		&self,
+		hash: Block::Hash,
+		header: Block::Header,
+		proof: StorageProof,
+	) -> ClientResult<AuthorityList> {
+		let storage_key = GRANDPA_AUTHORITIES_KEY.to_vec();
+		let request = RemoteReadRequest {
+			block: hash,
+			header,
+			keys: vec![storage_key.clone()],
+			retry_count: None,
+		};
+
+		self.check_read_proof(&request, proof)
+			.and_then(|results| {
+				let maybe_encoded = results.get(&storage_key)
+					.expect(
+						"storage_key is listed in the request keys; \
+						check_read_proof must return a value for each requested key;
+						qed"
+					);
+				maybe_encoded
+					.as_ref()
+					.and_then(|encoded| {
+						VersionedAuthorityList::decode(&mut encoded.as_slice()).ok()
+					})
+					.map(|versioned| versioned.into())
+					.ok_or(ClientError::InvalidAuthoritiesSet)
+			})
+	}
+}
+
+/// Finality proof provider for serving network requests.
+pub struct FinalityProofProvider<B,  Block: BlockT<Hash=H256>> {
+	backend: Arc<B>,
+	authority_provider: Arc<dyn AuthoritySetForFinalityProver<Block>>,
+}
+
+impl<B, Block: BlockT<Hash=H256>> FinalityProofProvider<B, Block>
+	where B: Backend<Block, Blake2Hasher> + Send + Sync + 'static
+{
+	/// Create new finality proof provider using:
+	///
+	/// - backend for accessing blockchain data;
+	/// - authority_provider for calling and proving runtime methods.
+	pub fn new(
+		backend: Arc<B>,
+		authority_provider: Arc<dyn AuthoritySetForFinalityProver<Block>>,
+	) -> Self {
+		FinalityProofProvider { backend, authority_provider }
+	}
+}
+
+impl<B, Block> network::FinalityProofProvider<Block> for FinalityProofProvider<B, Block>
+	where
+		Block: BlockT<Hash=H256>,
+		NumberFor<Block>: BlockNumberOps,
+		B: Backend<Block, Blake2Hasher> + Send + Sync + 'static,
+{
+	fn prove_finality(
+		&self,
+		for_block: Block::Hash,
+		request: &[u8],
+	) -> Result<Option<Vec<u8>>, ClientError> {
+		let request: FinalityProofRequest<Block::Hash> = Decode::decode(&mut &request[..])
+			.map_err(|e| {
+				warn!(target: "finality", "Unable to decode finality proof request: {}", e.what());
+				ClientError::Backend(format!("Invalid finality proof request"))
+			})?;
+		match request {
+			FinalityProofRequest::Original(request) => prove_finality::<_, _, GrandpaJustification<Block>>(
+				&*self.backend.blockchain(),
+				&*self.authority_provider,
+				request.authorities_set_id,
+				request.last_finalized,
+				for_block,
+			),
+		}
+	}
+}
+
+/// The effects of block finality.
+#[derive(Debug, PartialEq)]
+pub struct FinalityEffects<Header: HeaderT> {
+	/// The (ordered) set of headers that could be imported.
+	pub headers_to_import: Vec<Header>,
+	/// The hash of the block that could be finalized.
+	pub block: Header::Hash,
+	/// The justification for the block.
+	pub justification: Vec<u8>,
+	/// New authorities set id that should be applied starting from block.
+	pub new_set_id: u64,
+	/// New authorities set that should be applied starting from block.
+	pub new_authorities: AuthorityList,
+}
+
+/// Single fragment of proof-of-finality.
+///
+/// Finality for block B is proved by providing:
+/// 1) the justification for the descendant block F;
+/// 2) headers sub-chain (B; F] if B != F;
+/// 3) proof of GRANDPA::authorities() if the set changes at block F.
+#[derive(Debug, PartialEq, Encode, Decode)]
+struct FinalityProofFragment<Header: HeaderT> {
+	/// The hash of block F for which justification is provided.
+	pub block: Header::Hash,
+	/// Justification of the block F.
+	pub justification: Vec<u8>,
+	/// The set of headers in the range (U; F] that we believe are unknown to the caller. Ordered.
+	pub unknown_headers: Vec<Header>,
+	/// Optional proof of execution of GRANDPA::authorities().
+	pub authorities_proof: Option<StorageProof>,
+}
+
+/// Proof of finality is the ordered set of finality fragments, where:
+/// - last fragment provides justification for the best possible block from the requested range;
+/// - all other fragments provide justifications for GRANDPA authorities set changes within requested range.
+type FinalityProof<Header> = Vec<FinalityProofFragment<Header>>;
+
+/// Finality proof request data.
+#[derive(Debug, Encode, Decode)]
+enum FinalityProofRequest<H: Encode + Decode> {
+	/// Original version of the request.
+	Original(OriginalFinalityProofRequest<H>),
+}
+
+/// Original version of finality proof request.
+#[derive(Debug, Encode, Decode)]
+struct OriginalFinalityProofRequest<H: Encode + Decode> {
+	/// The authorities set id we are waiting proof from.
+	///
+	/// The first justification in the proof must be signed by this authority set.
+	pub authorities_set_id: u64,
+	/// Hash of the last known finalized block.
+	pub last_finalized: H,
+}
+
+/// Prepare data blob associated with finality proof request.
+pub(crate) fn make_finality_proof_request<H: Encode + Decode>(last_finalized: H, authorities_set_id: u64) -> Vec<u8> {
+	FinalityProofRequest::Original(OriginalFinalityProofRequest {
+		authorities_set_id,
+		last_finalized,
+	}).encode()
+}
+
+/// Prepare proof-of-finality for the best possible block in the range: (begin; end].
+///
+/// It is assumed that the caller already have a proof-of-finality for the block 'begin'.
+/// It is assumed that the caller already knows all blocks in the range (begin; end].
+///
+/// Returns None if there are no finalized blocks unknown to the caller.
+pub(crate) fn prove_finality<Block: BlockT<Hash=H256>, B: BlockchainBackend<Block>, J>(
+	blockchain: &B,
+	authorities_provider: &dyn AuthoritySetForFinalityProver<Block>,
+	authorities_set_id: u64,
+	begin: Block::Hash,
+	end: Block::Hash,
+) -> ::client_api::error::Result<Option<Vec<u8>>>
+	where
+		J: ProvableJustification<Block::Header>,
+{
+	let begin_id = BlockId::Hash(begin);
+	let begin_number = blockchain.expect_block_number_from_id(&begin_id)?;
+
+	// early-return if we sure that there are no blocks finalized AFTER begin block
+	let info = blockchain.info();
+	if info.finalized_number <= begin_number {
+		trace!(
+			target: "finality",
+			"Requested finality proof for descendant of #{} while we only have finalized #{}. Returning empty proof.",
+			begin_number,
+			info.finalized_number,
+		);
+
+		return Ok(None);
+	}
+
+	// check if blocks range is valid. It is the caller responsibility to ensure
+	// that it only asks peers that know about whole blocks range
+	let end_number = blockchain.expect_block_number_from_id(&BlockId::Hash(end))?;
+	if begin_number + One::one() > end_number {
+		return Err(ClientError::Backend(
+			format!("Cannot generate finality proof for invalid range: {}..{}", begin_number, end_number),
+		));
+	}
+
+	// early-return if we sure that the block is NOT a part of canonical chain
+	let canonical_begin = blockchain.expect_block_hash_from_id(&BlockId::Number(begin_number))?;
+	if begin != canonical_begin {
+		return Err(ClientError::Backend(
+			format!("Cannot generate finality proof for non-canonical block: {}", begin),
+		));
+	}
+
+	// iterate justifications && try to prove finality
+	let mut fragment_index = 0;
+	let mut current_authorities = authorities_provider.authorities(&begin_id)?;
+	let mut current_number = begin_number + One::one();
+	let mut finality_proof = Vec::new();
+	let mut unknown_headers = Vec::new();
+	let mut latest_proof_fragment = None;
+	loop {
+		let current_id = BlockId::Number(current_number);
+
+		// check if header is unknown to the caller
+		if current_number > end_number {
+			let unknown_header = blockchain.expect_header(current_id)?;
+			unknown_headers.push(unknown_header);
+		}
+
+		if let Some(justification) = blockchain.justification(current_id)? {
+			// check if the current block enacts new GRANDPA authorities set
+			let parent_id = BlockId::Number(current_number - One::one());
+			let new_authorities = authorities_provider.authorities(&parent_id)?;
+			let new_authorities_proof = if current_authorities != new_authorities {
+				current_authorities = new_authorities;
+				Some(authorities_provider.prove_authorities(&parent_id)?)
+			} else {
+				None
+			};
+
+			// prepare finality proof for the current block
+			let current = blockchain.expect_block_hash_from_id(&BlockId::Number(current_number))?;
+			let proof_fragment = FinalityProofFragment {
+				block: current,
+				justification,
+				unknown_headers: ::std::mem::replace(&mut unknown_headers, Vec::new()),
+				authorities_proof: new_authorities_proof,
+			};
+
+			// append justification to finality proof if required
+			let justifies_end_block = current_number >= end_number;
+			let justifies_authority_set_change = proof_fragment.authorities_proof.is_some();
+			if justifies_end_block || justifies_authority_set_change {
+				// check if the proof is generated by the requested authority set
+				if finality_proof.is_empty() {
+					let justification_check_result = J::decode_and_verify(
+						&proof_fragment.justification,
+						authorities_set_id,
+						&current_authorities,
+					);
+					if justification_check_result.is_err() {
+						trace!(
+							target: "finality",
+							"Can not provide finality proof with requested set id #{}\
+							(possible forced change?). Returning empty proof.",
+							authorities_set_id,
+						);
+
+						return Ok(None);
+					}
+				}
+
+				finality_proof.push(proof_fragment);
+				latest_proof_fragment = None;
+			} else {
+				latest_proof_fragment = Some(proof_fragment);
+			}
+
+			// we don't need to provide more justifications
+			if justifies_end_block {
+				break;
+			}
+		}
+
+		// we can't provide more justifications
+		if current_number == info.finalized_number {
+			// append last justification - even if we can't generate finality proof for
+			// the end block, we try to generate it for the latest possible block
+			if let Some(latest_proof_fragment) = latest_proof_fragment.take() {
+				finality_proof.push(latest_proof_fragment);
+
+				fragment_index += 1;
+				if fragment_index == MAX_FRAGMENTS_IN_PROOF {
+					break;
+				}
+			}
+			break;
+		}
+
+		// else search for the next justification
+		current_number = current_number + One::one();
+	}
+
+	if finality_proof.is_empty() {
+		trace!(
+			target: "finality",
+			"No justifications found when making finality proof for {}. Returning empty proof.",
+			end,
+		);
+
+		Ok(None)
+	} else {
+		trace!(
+			target: "finality",
+			"Built finality proof for {} of {} fragments. Last fragment for {}.",
+			end,
+			finality_proof.len(),
+			finality_proof.last().expect("checked that !finality_proof.is_empty(); qed").block,
+		);
+
+		Ok(Some(finality_proof.encode()))
+	}
+}
+
+/// Check GRANDPA proof-of-finality for the given block.
+///
+/// Returns the vector of headers that MUST be validated + imported
+/// AND if at least one of those headers is invalid, all other MUST be considered invalid.
+pub(crate) fn check_finality_proof<Block: BlockT<Hash=H256>, B>(
+	blockchain: &B,
+	current_set_id: u64,
+	current_authorities: AuthorityList,
+	authorities_provider: &dyn AuthoritySetForFinalityChecker<Block>,
+	remote_proof: Vec<u8>,
+) -> ClientResult<FinalityEffects<Block::Header>>
+	where
+		NumberFor<Block>: BlockNumberOps,
+		B: BlockchainBackend<Block>,
+{
+	do_check_finality_proof::<_, _, GrandpaJustification<Block>>(
+		blockchain,
+		current_set_id,
+		current_authorities,
+		authorities_provider,
+		remote_proof)
+}
+
+fn do_check_finality_proof<Block: BlockT<Hash=H256>, B, J>(
+	blockchain: &B,
+	current_set_id: u64,
+	current_authorities: AuthorityList,
+	authorities_provider: &dyn AuthoritySetForFinalityChecker<Block>,
+	remote_proof: Vec<u8>,
+) -> ClientResult<FinalityEffects<Block::Header>>
+	where
+		NumberFor<Block>: BlockNumberOps,
+		B: BlockchainBackend<Block>,
+		J: ProvableJustification<Block::Header>,
+{
+	// decode finality proof
+	let proof = FinalityProof::<Block::Header>::decode(&mut &remote_proof[..])
+		.map_err(|_| ClientError::BadJustification("failed to decode finality proof".into()))?;
+
+	// empty proof can't prove anything
+	if proof.is_empty() {
+		return Err(ClientError::BadJustification("empty proof of finality".into()));
+	}
+
+	// iterate and verify proof fragments
+	let last_fragment_index = proof.len() - 1;
+	let mut authorities = AuthoritiesOrEffects::Authorities(current_set_id, current_authorities);
+	for (proof_fragment_index, proof_fragment) in proof.into_iter().enumerate() {
+		// check that proof is non-redundant. The proof still can be valid, but
+		// we do not want peer to spam us with redundant data
+		if proof_fragment_index != last_fragment_index {
+			let has_unknown_headers = !proof_fragment.unknown_headers.is_empty();
+			let has_new_authorities = proof_fragment.authorities_proof.is_some();
+			if has_unknown_headers || !has_new_authorities {
+				return Err(ClientError::BadJustification("redundant proof of finality".into()));
+			}
+		}
+
+		authorities = check_finality_proof_fragment::<_, _, J>(
+			blockchain,
+			authorities,
+			authorities_provider,
+			proof_fragment)?;
+	}
+
+	let effects = authorities.extract_effects().expect("at least one loop iteration is guaranteed
+			because proof is not empty;\
+			check_finality_proof_fragment is called on every iteration;\
+			check_finality_proof_fragment always returns FinalityEffects;\
+			qed");
+
+	telemetry!(CONSENSUS_INFO; "afg.finality_proof_ok";
+		"set_id" => ?effects.new_set_id, "finalized_header_hash" => ?effects.block);
+
+	Ok(effects)
+}
+
+/// Check finality proof for the single block.
+fn check_finality_proof_fragment<Block: BlockT<Hash=H256>, B, J>(
+	blockchain: &B,
+	authority_set: AuthoritiesOrEffects<Block::Header>,
+	authorities_provider: &dyn AuthoritySetForFinalityChecker<Block>,
+	proof_fragment: FinalityProofFragment<Block::Header>,
+) -> ClientResult<AuthoritiesOrEffects<Block::Header>>
+	where
+		NumberFor<Block>: BlockNumberOps,
+		B: BlockchainBackend<Block>,
+		J: Decode + ProvableJustification<Block::Header>,
+{
+	// verify justification using previous authorities set
+	let (mut current_set_id, mut current_authorities) = authority_set.extract_authorities();
+	let justification: J = Decode::decode(&mut &proof_fragment.justification[..])
+		.map_err(|_| ClientError::JustificationDecode)?;
+	justification.verify(current_set_id, &current_authorities)?;
+
+	// and now verify new authorities proof (if provided)
+	if let Some(new_authorities_proof) = proof_fragment.authorities_proof {
+		// it is safe to query header here, because its non-finality proves that it can't be pruned
+		let header = blockchain.expect_header(BlockId::Hash(proof_fragment.block))?;
+		let parent_hash = *header.parent_hash();
+		let parent_header = blockchain.expect_header(BlockId::Hash(parent_hash))?;
+		current_authorities = authorities_provider.check_authorities_proof(
+			parent_hash,
+			parent_header,
+			new_authorities_proof,
+		)?;
+
+		current_set_id = current_set_id + 1;
+	}
+
+	Ok(AuthoritiesOrEffects::Effects(FinalityEffects {
+		headers_to_import: proof_fragment.unknown_headers,
+		block: proof_fragment.block,
+		justification: proof_fragment.justification,
+		new_set_id: current_set_id,
+		new_authorities: current_authorities,
+	}))
+}
+
+/// Authorities set from initial authorities set or finality effects.
+enum AuthoritiesOrEffects<Header: HeaderT> {
+	Authorities(u64, AuthorityList),
+	Effects(FinalityEffects<Header>),
+}
+
+impl<Header: HeaderT> AuthoritiesOrEffects<Header> {
+	pub fn extract_authorities(self) -> (u64, AuthorityList) {
+		match self {
+			AuthoritiesOrEffects::Authorities(set_id, authorities) => (set_id, authorities),
+			AuthoritiesOrEffects::Effects(effects) => (effects.new_set_id, effects.new_authorities),
+		}
+	}
+
+	pub fn extract_effects(self) -> Option<FinalityEffects<Header>> {
+		match self {
+			AuthoritiesOrEffects::Authorities(_, _) => None,
+			AuthoritiesOrEffects::Effects(effects) => Some(effects),
+		}
+	}
+}
+
+/// Justification used to prove block finality.
+pub(crate) trait ProvableJustification<Header: HeaderT>: Encode + Decode {
+	/// Verify justification with respect to authorities set and authorities set id.
+	fn verify(&self, set_id: u64, authorities: &[(AuthorityId, u64)]) -> ClientResult<()>;
+
+	/// Decode and verify justification.
+	fn decode_and_verify(
+		justification: &Justification,
+		set_id: u64,
+		authorities: &[(AuthorityId, u64)],
+	) -> ClientResult<Self> {
+		let justification = Self::decode(&mut &**justification)
+			.map_err(|_| ClientError::JustificationDecode)?;
+		justification.verify(set_id, authorities)?;
+		Ok(justification)
+	}
+}
+
+impl<Block: BlockT<Hash=H256>> ProvableJustification<Block::Header> for GrandpaJustification<Block>
+	where
+		NumberFor<Block>: BlockNumberOps,
+{
+	fn verify(&self, set_id: u64, authorities: &[(AuthorityId, u64)]) -> ClientResult<()> {
+		GrandpaJustification::verify(self, set_id, &authorities.iter().cloned().collect())
+	}
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+	use test_client::runtime::{Block, Header, H256};
+	use client_api::NewBlockState;
+	use test_client::client::in_mem::Blockchain as InMemoryBlockchain;
+	use super::*;
+	use primitives::crypto::Public;
+
+	type FinalityProof = super::FinalityProof<Header>;
+
+	impl<GetAuthorities, ProveAuthorities> AuthoritySetForFinalityProver<Block> for (GetAuthorities, ProveAuthorities)
+		where
+			GetAuthorities: Send + Sync + Fn(BlockId<Block>) -> ClientResult<AuthorityList>,
+			ProveAuthorities: Send + Sync + Fn(BlockId<Block>) -> ClientResult<StorageProof>,
+	{
+		fn authorities(&self, block: &BlockId<Block>) -> ClientResult<AuthorityList> {
+			self.0(*block)
+		}
+
+		fn prove_authorities(&self, block: &BlockId<Block>) -> ClientResult<StorageProof> {
+			self.1(*block)
+		}
+	}
+
+	struct ClosureAuthoritySetForFinalityChecker<Closure>(pub Closure);
+
+	impl<Closure> AuthoritySetForFinalityChecker<Block> for ClosureAuthoritySetForFinalityChecker<Closure>
+		where
+			Closure: Send + Sync + Fn(H256, Header, StorageProof) -> ClientResult<AuthorityList>,
+	{
+		fn check_authorities_proof(
+			&self,
+			hash: H256,
+			header: Header,
+			proof: StorageProof
+		) -> ClientResult<AuthorityList> {
+			self.0(hash, header, proof)
+		}
+	}
+
+	#[derive(Debug, PartialEq, Encode, Decode)]
+	pub struct TestJustification(pub bool, pub Vec<u8>);
+
+	impl ProvableJustification<Header> for TestJustification {
+		fn verify(&self, _set_id: u64, _authorities: &[(AuthorityId, u64)]) -> ClientResult<()> {
+			if self.0 {
+				Ok(())
+			} else {
+				Err(ClientError::BadJustification("test".into()))
+			}
+		}
+	}
+
+	fn header(number: u64) -> Header {
+		let parent_hash = match number {
+			0 => Default::default(),
+			_ => header(number - 1).hash(),
+		};
+		Header::new(number, H256::from_low_u64_be(0), H256::from_low_u64_be(0), parent_hash, Default::default())
+	}
+
+	fn side_header(number: u64) -> Header {
+		Header::new(
+			number,
+			H256::from_low_u64_be(0),
+			H256::from_low_u64_be(1),
+			header(number - 1).hash(),
+			Default::default(),
+		)
+	}
+
+	fn second_side_header(number: u64) -> Header {
+		Header::new(
+			number,
+			H256::from_low_u64_be(0),
+			H256::from_low_u64_be(1),
+			side_header(number - 1).hash(),
+			Default::default(),
+		)
+	}
+
+	fn test_blockchain() -> InMemoryBlockchain<Block> {
+		let blockchain = InMemoryBlockchain::<Block>::new();
+		blockchain.insert(header(0).hash(), header(0), Some(vec![0]), None, NewBlockState::Final).unwrap();
+		blockchain.insert(header(1).hash(), header(1), Some(vec![1]), None, NewBlockState::Final).unwrap();
+		blockchain.insert(header(2).hash(), header(2), None, None, NewBlockState::Best).unwrap();
+		blockchain.insert(header(3).hash(), header(3), Some(vec![3]), None, NewBlockState::Final).unwrap();
+		blockchain
+	}
+
+	#[test]
+	fn finality_prove_fails_with_invalid_range() {
+		let blockchain = test_blockchain();
+
+		// their last finalized is: 2
+		// they request for proof-of-finality of: 2
+		// => range is invalid
+		prove_finality::<_, _, TestJustification>(
+			&blockchain,
+			&(
+				|_| unreachable!("should return before calling GetAuthorities"),
+				|_| unreachable!("should return before calling ProveAuthorities"),
+			),
+			0,
+			header(2).hash(),
+			header(2).hash(),
+		).unwrap_err();
+	}
+
+	#[test]
+	fn finality_proof_is_none_if_no_more_last_finalized_blocks() {
+		let blockchain = test_blockchain();
+		blockchain.insert(header(4).hash(), header(4), None, None, NewBlockState::Best).unwrap();
+
+		// our last finalized is: 3
+		// their last finalized is: 3
+		// => we can't provide any additional justifications
+		let proof_of_4 = prove_finality::<_, _, TestJustification>(
+			&blockchain,
+			&(
+				|_| unreachable!("should return before calling GetAuthorities"),
+				|_| unreachable!("should return before calling ProveAuthorities"),
+			),
+			0,
+			header(3).hash(),
+			header(4).hash(),
+		).unwrap();
+		assert_eq!(proof_of_4, None);
+	}
+
+	#[test]
+	fn finality_proof_fails_for_non_canonical_block() {
+		let blockchain = test_blockchain();
+		blockchain.insert(header(4).hash(), header(4), None, None, NewBlockState::Best).unwrap();
+		blockchain.insert(side_header(4).hash(), side_header(4), None, None, NewBlockState::Best).unwrap();
+		blockchain.insert(second_side_header(5).hash(), second_side_header(5), None, None, NewBlockState::Best)
+			.unwrap();
+		blockchain.insert(header(5).hash(), header(5), Some(vec![5]), None, NewBlockState::Final).unwrap();
+
+		// chain is 1 -> 2 -> 3 -> 4 -> 5
+		//                      \> 4' -> 5'
+		// and the best finalized is 5
+		// => when requesting for (4'; 5'], error is returned
+		prove_finality::<_, _, TestJustification>(
+			&blockchain,
+			&(
+				|_| unreachable!("should return before calling GetAuthorities"),
+				|_| unreachable!("should return before calling ProveAuthorities"),
+			),
+			0,
+			side_header(4).hash(),
+			second_side_header(5).hash(),
+		).unwrap_err();
+	}
+
+	#[test]
+	fn finality_proof_is_none_if_no_justification_known() {
+		let blockchain = test_blockchain();
+		blockchain.insert(header(4).hash(), header(4), None, None, NewBlockState::Final).unwrap();
+
+		// block 4 is finalized without justification
+		// => we can't prove finality
+		let proof_of_4 = prove_finality::<_, _, TestJustification>(
+			&blockchain,
+			&(
+				|_| Ok(vec![(AuthorityId::from_slice(&[1u8; 32]), 1u64)]),
+				|_| unreachable!("authorities didn't change => ProveAuthorities won't be called"),
+			),
+			0,
+			header(3).hash(),
+			header(4).hash(),
+		).unwrap();
+		assert_eq!(proof_of_4, None);
+	}
+
+	#[test]
+	fn finality_proof_works_without_authorities_change() {
+		let blockchain = test_blockchain();
+		let just4 = TestJustification(true, vec![4]).encode();
+		let just5 = TestJustification(true, vec![5]).encode();
+		blockchain.insert(header(4).hash(), header(4), Some(just4), None, NewBlockState::Final).unwrap();
+		blockchain.insert(header(5).hash(), header(5), Some(just5.clone()), None, NewBlockState::Final).unwrap();
+
+		// blocks 4 && 5 are finalized with justification
+		// => since authorities are the same, we only need justification for 5
+		let proof_of_5: FinalityProof = Decode::decode(&mut &prove_finality::<_, _, TestJustification>(
+			&blockchain,
+			&(
+				|_| Ok(vec![(AuthorityId::from_slice(&[1u8; 32]), 1u64)]),
+				|_| unreachable!("should return before calling ProveAuthorities"),
+			),
+			0,
+			header(3).hash(),
+			header(5).hash(),
+		).unwrap().unwrap()[..]).unwrap();
+		assert_eq!(proof_of_5, vec![FinalityProofFragment {
+			block: header(5).hash(),
+			justification: just5,
+			unknown_headers: Vec::new(),
+			authorities_proof: None,
+		}]);
+	}
+
+	#[test]
+	fn finality_proof_finalized_earlier_block_if_no_justification_for_target_is_known() {
+		let blockchain = test_blockchain();
+		blockchain.insert(header(4).hash(), header(4), Some(vec![4]), None, NewBlockState::Final).unwrap();
+		blockchain.insert(header(5).hash(), header(5), None, None, NewBlockState::Final).unwrap();
+
+		// block 4 is finalized with justification + we request for finality of 5
+		// => we can't prove finality of 5, but providing finality for 4 is still useful for requester
+		let proof_of_5: FinalityProof = Decode::decode(&mut &prove_finality::<_, _, TestJustification>(
+			&blockchain,
+			&(
+				|_| Ok(vec![(AuthorityId::from_slice(&[1u8; 32]), 1u64)]),
+				|_| unreachable!("should return before calling ProveAuthorities"),
+			),
+			0,
+			header(3).hash(),
+			header(5).hash(),
+		).unwrap().unwrap()[..]).unwrap();
+		assert_eq!(proof_of_5, vec![FinalityProofFragment {
+			block: header(4).hash(),
+			justification: vec![4],
+			unknown_headers: Vec::new(),
+			authorities_proof: None,
+		}]);
+	}
+
+	#[test]
+	fn finality_proof_works_with_authorities_change() {
+		let blockchain = test_blockchain();
+		let just4 = TestJustification(true, vec![4]).encode();
+		let just5 = TestJustification(true, vec![5]).encode();
+		let just7 = TestJustification(true, vec![7]).encode();
+		blockchain.insert(header(4).hash(), header(4), Some(just4), None, NewBlockState::Final).unwrap();
+		blockchain.insert(header(5).hash(), header(5), Some(just5.clone()), None, NewBlockState::Final).unwrap();
+		blockchain.insert(header(6).hash(), header(6), None, None, NewBlockState::Final).unwrap();
+		blockchain.insert(header(7).hash(), header(7), Some(just7.clone()), None, NewBlockState::Final).unwrap();
+
+		// when querying for finality of 6, we assume that the #6 is the last block known to the requester
+		// => since we only have justification for #7, we provide #7
+		let proof_of_6: FinalityProof = Decode::decode(&mut &prove_finality::<_, _, TestJustification>(
+			&blockchain,
+			&(
+				|block_id| match block_id {
+					BlockId::Hash(h) if h == header(3).hash() => Ok(
+						vec![(AuthorityId::from_slice(&[3u8; 32]), 1u64)]
+					),
+					BlockId::Number(3) => Ok(vec![(AuthorityId::from_slice(&[3u8; 32]), 1u64)]),
+					BlockId::Number(4) => Ok(vec![(AuthorityId::from_slice(&[4u8; 32]), 1u64)]),
+					BlockId::Number(6) => Ok(vec![(AuthorityId::from_slice(&[6u8; 32]), 1u64)]),
+					_ => unreachable!("no other authorities should be fetched: {:?}", block_id),
+				},
+				|block_id| match block_id {
+					BlockId::Number(4) => Ok(StorageProof::new(vec![vec![40]])),
+					BlockId::Number(6) => Ok(StorageProof::new(vec![vec![60]])),
+					_ => unreachable!("no other authorities should be proved: {:?}", block_id),
+				},
+			),
+			0,
+			header(3).hash(),
+			header(6).hash(),
+		).unwrap().unwrap()[..]).unwrap();
+		// initial authorities set (which start acting from #4) is [3; 32]
+		assert_eq!(proof_of_6, vec![
+			// new authorities set starts acting from #5 => we do not provide fragment for #4
+			// first fragment provides justification for #5 && authorities set that starts acting from #5
+			FinalityProofFragment {
+				block: header(5).hash(),
+				justification: just5,
+				unknown_headers: Vec::new(),
+				authorities_proof: Some(StorageProof::new(vec![vec![40]])),
+			},
+			// last fragment provides justification for #7 && unknown#7
+			FinalityProofFragment {
+				block: header(7).hash(),
+				justification: just7,
+				unknown_headers: vec![header(7)],
+				authorities_proof: Some(StorageProof::new(vec![vec![60]])),
+			},
+		]);
+	}
+
+	#[test]
+	fn finality_proof_check_fails_when_proof_decode_fails() {
+		let blockchain = test_blockchain();
+
+		// when we can't decode proof from Vec<u8>
+		do_check_finality_proof::<_, _, TestJustification>(
+			&blockchain,
+			1,
+			vec![(AuthorityId::from_slice(&[3u8; 32]), 1u64)],
+			&ClosureAuthoritySetForFinalityChecker(|_, _, _| unreachable!("returns before CheckAuthoritiesProof")),
+			vec![42],
+		).unwrap_err();
+	}
+
+	#[test]
+	fn finality_proof_check_fails_when_proof_is_empty() {
+		let blockchain = test_blockchain();
+
+		// when decoded proof has zero length
+		do_check_finality_proof::<_, _, TestJustification>(
+			&blockchain,
+			1,
+			vec![(AuthorityId::from_slice(&[3u8; 32]), 1u64)],
+			&ClosureAuthoritySetForFinalityChecker(|_, _, _| unreachable!("returns before CheckAuthoritiesProof")),
+			Vec::<TestJustification>::new().encode(),
+		).unwrap_err();
+	}
+
+	#[test]
+	fn finality_proof_check_fails_when_intemediate_fragment_has_unknown_headers() {
+		let blockchain = test_blockchain();
+
+		// when intermediate (#0) fragment has non-empty unknown headers
+		do_check_finality_proof::<_, _, TestJustification>(
+			&blockchain,
+			1,
+			vec![(AuthorityId::from_slice(&[3u8; 32]), 1u64)],
+			&ClosureAuthoritySetForFinalityChecker(|_, _, _| unreachable!("returns before CheckAuthoritiesProof")),
+			vec![FinalityProofFragment {
+				block: header(4).hash(),
+				justification: TestJustification(true, vec![7]).encode(),
+				unknown_headers: vec![header(4)],
+				authorities_proof: Some(StorageProof::new(vec![vec![42]])),
+			}, FinalityProofFragment {
+				block: header(5).hash(),
+				justification: TestJustification(true, vec![8]).encode(),
+				unknown_headers: vec![header(5)],
+				authorities_proof: None,
+			}].encode(),
+		).unwrap_err();
+	}
+
+	#[test]
+	fn finality_proof_check_fails_when_intemediate_fragment_has_no_authorities_proof() {
+		let blockchain = test_blockchain();
+
+		// when intermediate (#0) fragment has empty authorities proof
+		do_check_finality_proof::<_, _, TestJustification>(
+			&blockchain,
+			1,
+			vec![(AuthorityId::from_slice(&[3u8; 32]), 1u64)],
+			&ClosureAuthoritySetForFinalityChecker(|_, _, _| unreachable!("returns before CheckAuthoritiesProof")),
+			vec![FinalityProofFragment {
+				block: header(4).hash(),
+				justification: TestJustification(true, vec![7]).encode(),
+				unknown_headers: Vec::new(),
+				authorities_proof: None,
+			}, FinalityProofFragment {
+				block: header(5).hash(),
+				justification: TestJustification(true, vec![8]).encode(),
+				unknown_headers: vec![header(5)],
+				authorities_proof: None,
+			}].encode(),
+		).unwrap_err();
+	}
+
+	#[test]
+	fn finality_proof_check_works() {
+		let blockchain = test_blockchain();
+
+		let effects = do_check_finality_proof::<_, _, TestJustification>(
+			&blockchain,
+			1,
+			vec![(AuthorityId::from_slice(&[3u8; 32]), 1u64)],
+			&ClosureAuthoritySetForFinalityChecker(|_, _, _| Ok(vec![(AuthorityId::from_slice(&[4u8; 32]), 1u64)])),
+			vec![FinalityProofFragment {
+				block: header(2).hash(),
+				justification: TestJustification(true, vec![7]).encode(),
+				unknown_headers: Vec::new(),
+				authorities_proof: Some(StorageProof::new(vec![vec![42]])),
+			}, FinalityProofFragment {
+				block: header(4).hash(),
+				justification: TestJustification(true, vec![8]).encode(),
+				unknown_headers: vec![header(4)],
+				authorities_proof: None,
+			}].encode(),
+		).unwrap();
+		assert_eq!(effects, FinalityEffects {
+			headers_to_import: vec![header(4)],
+			block: header(4).hash(),
+			justification: TestJustification(true, vec![8]).encode(),
+			new_set_id: 2,
+			new_authorities: vec![(AuthorityId::from_slice(&[4u8; 32]), 1u64)],
+		});
+	}
+
+	#[test]
+	fn finality_proof_is_none_if_first_justification_is_generated_by_unknown_set() {
+		// this is the case for forced change: set_id has been forcibly increased on full node
+		// and ligh node missed that
+		// => justification verification will fail on light node anyways, so we do not return
+		// finality proof at all
+		let blockchain = test_blockchain();
+		let just4 = TestJustification(false, vec![4]).encode(); // false makes verification fail
+		blockchain.insert(header(4).hash(), header(4), Some(just4), None, NewBlockState::Final).unwrap();
+
+		let proof_of_4 = prove_finality::<_, _, TestJustification>(
+			&blockchain,
+			&(
+				|_| Ok(vec![(AuthorityId::from_slice(&[1u8; 32]), 1u64)]),
+				|_| unreachable!("should return before calling ProveAuthorities"),
+			),
+			0,
+			header(3).hash(),
+			header(4).hash(),
+		).unwrap();
+		assert!(proof_of_4.is_none());
+	}
+}