Reorganising the repository - external renames and moves (#4074)

* Adding first rough ouline of the repository structure

* Remove old CI stuff

* add title

* formatting fixes

* move node-exits job's script to scripts dir

* Move docs into subdir

* move to bin

* move maintainence scripts, configs and helpers into its own dir

* add .local to ignore

* move core->client

* start up 'test' area

* move test client

* move test runtime

* make test move compile

* Add dependencies rule enforcement.

* Fix indexing.

* Update docs to reflect latest changes

* Moving /srml->/paint

* update docs

* move client/sr-* -> primitives/

* clean old readme

* remove old broken code in rhd

* update lock

* Step 1.

* starting to untangle client

* Fix after merge.

* start splitting out client interfaces

* move children and blockchain interfaces

* Move trie and state-machine to primitives.

* Fix WASM builds.

* fixing broken imports

* more interface moves

* move backend and light to interfaces

* move CallExecutor

* move cli off client

* moving around more interfaces

* re-add consensus crates into the mix

* fix subkey path

* relieve client from executor

* starting to pull out client from grandpa

* move is_decendent_of out of client

* grandpa still depends on client directly

* lemme tests pass

* rename srml->paint

* Make it compile.

* rename interfaces->client-api

* Move keyring to primitives.

* fixup libp2p dep

* fix broken use

* allow dependency enforcement to fail

* move fork-tree

* Moving wasm-builder

* make env

* move build-script-utils

* fixup broken crate depdencies and names

* fix imports for authority discovery

* fix typo

* update cargo.lock

* fixing imports

* Fix paths and add missing crates

* re-add missing crates

substrate/client/consensus/babe/src/authorship.rs

@@ -0,0 +1,214 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! BABE authority selection and slot claiming.
+
+use merlin::Transcript;
+use babe_primitives::{AuthorityId, BabeAuthorityWeight, BABE_ENGINE_ID, BABE_VRF_PREFIX};
+use babe_primitives::{Epoch, SlotNumber, AuthorityPair, BabePreDigest, BabeConfiguration};
+use primitives::{U256, blake2_256};
+use codec::Encode;
+use schnorrkel::vrf::VRFInOut;
+use primitives::Pair;
+use keystore::KeyStorePtr;
+
+/// Calculates the primary selection threshold for a given authority, taking
+/// into account `c` (`1 - c` represents the probability of a slot being empty).
+pub(super) fn calculate_primary_threshold(
+	c: (u64, u64),
+	authorities: &[(AuthorityId, BabeAuthorityWeight)],
+	authority_index: usize,
+) -> u128 {
+	use num_bigint::BigUint;
+	use num_rational::BigRational;
+	use num_traits::{cast::ToPrimitive, identities::One};
+
+	let c = c.0 as f64 / c.1 as f64;
+
+	let theta =
+		authorities[authority_index].1 as f64 /
+		authorities.iter().map(|(_, weight)| weight).sum::<u64>() as f64;
+
+	let calc = || {
+		let p = BigRational::from_float(1f64 - (1f64 - c).powf(theta))?;
+		let numer = p.numer().to_biguint()?;
+		let denom = p.denom().to_biguint()?;
+		((BigUint::one() << 128) * numer / denom).to_u128()
+	};
+
+	calc().unwrap_or(u128::max_value())
+}
+
+/// Returns true if the given VRF output is lower than the given threshold,
+/// false otherwise.
+pub(super) fn check_primary_threshold(inout: &VRFInOut, threshold: u128) -> bool {
+	u128::from_le_bytes(inout.make_bytes::<[u8; 16]>(BABE_VRF_PREFIX)) < threshold
+}
+
+/// Get the expected secondary author for the given slot and with given
+/// authorities. This should always assign the slot to some authority unless the
+/// authorities list is empty.
+pub(super) fn secondary_slot_author(
+	slot_number: u64,
+	authorities: &[(AuthorityId, BabeAuthorityWeight)],
+	randomness: [u8; 32],
+) -> Option<&AuthorityId> {
+	if authorities.is_empty() {
+		return None;
+	}
+
+	let rand = U256::from((randomness, slot_number).using_encoded(blake2_256));
+
+	let authorities_len = U256::from(authorities.len());
+	let idx = rand % authorities_len;
+
+	let expected_author = authorities.get(idx.as_u32() as usize)
+		.expect("authorities not empty; index constrained to list length; \
+				this is a valid index; qed");
+
+	Some(&expected_author.0)
+}
+
+#[allow(deprecated)]
+pub(super) fn make_transcript(
+	randomness: &[u8],
+	slot_number: u64,
+	epoch: u64,
+) -> Transcript {
+	let mut transcript = Transcript::new(&BABE_ENGINE_ID);
+	transcript.commit_bytes(b"slot number", &slot_number.to_le_bytes());
+	transcript.commit_bytes(b"current epoch", &epoch.to_le_bytes());
+	transcript.commit_bytes(b"chain randomness", randomness);
+	transcript
+}
+
+
+/// Claim a secondary slot if it is our turn to propose, returning the
+/// pre-digest to use when authoring the block, or `None` if it is not our turn
+/// to propose.
+fn claim_secondary_slot(
+	slot_number: SlotNumber,
+	authorities: &[(AuthorityId, BabeAuthorityWeight)],
+	keystore: &KeyStorePtr,
+	randomness: [u8; 32],
+) -> Option<(BabePreDigest, AuthorityPair)> {
+	if authorities.is_empty() {
+		return None;
+	}
+
+	let expected_author = super::authorship::secondary_slot_author(
+		slot_number,
+		authorities,
+		randomness,
+	)?;
+
+	let keystore = keystore.read();
+
+	for (pair, authority_index) in authorities.iter()
+		.enumerate()
+		.flat_map(|(i, a)| {
+			keystore.key_pair::<AuthorityPair>(&a.0).ok().map(|kp| (kp, i))
+		})
+	{
+		if pair.public() == *expected_author {
+			let pre_digest = BabePreDigest::Secondary {
+				slot_number,
+				authority_index: authority_index as u32,
+			};
+
+			return Some((pre_digest, pair));
+		}
+	}
+
+	None
+}
+
+/// Tries to claim the given slot number. This method starts by trying to claim
+/// a primary VRF based slot. If we are not able to claim it, then if we have
+/// secondary slots enabled for the given epoch, we will fallback to trying to
+/// claim a secondary slot.
+pub(super) fn claim_slot(
+	slot_number: SlotNumber,
+	epoch: &Epoch,
+	config: &BabeConfiguration,
+	keystore: &KeyStorePtr,
+) -> Option<(BabePreDigest, AuthorityPair)> {
+	claim_primary_slot(slot_number, epoch, config.c, keystore)
+		.or_else(|| {
+			if config.secondary_slots {
+				claim_secondary_slot(
+					slot_number,
+					&epoch.authorities,
+					keystore,
+					epoch.randomness,
+				)
+			} else {
+				None
+			}
+		})
+}
+
+fn get_keypair(q: &AuthorityPair) -> &schnorrkel::Keypair {
+	use primitives::crypto::IsWrappedBy;
+	primitives::sr25519::Pair::from_ref(q).as_ref()
+}
+
+/// Claim a primary slot if it is our turn.  Returns `None` if it is not our turn.
+/// This hashes the slot number, epoch, genesis hash, and chain randomness into
+/// the VRF.  If the VRF produces a value less than `threshold`, it is our turn,
+/// so it returns `Some(_)`. Otherwise, it returns `None`.
+fn claim_primary_slot(
+	slot_number: SlotNumber,
+	epoch: &Epoch,
+	c: (u64, u64),
+	keystore: &KeyStorePtr,
+) -> Option<(BabePreDigest, AuthorityPair)> {
+	let Epoch { authorities, randomness, epoch_index, .. } = epoch;
+	let keystore = keystore.read();
+
+	for (pair, authority_index) in authorities.iter()
+		.enumerate()
+		.flat_map(|(i, a)| {
+			keystore.key_pair::<AuthorityPair>(&a.0).ok().map(|kp| (kp, i))
+		})
+	{
+		let transcript = super::authorship::make_transcript(randomness, slot_number, *epoch_index);
+
+		// Compute the threshold we will use.
+		//
+		// We already checked that authorities contains `key.public()`, so it can't
+		// be empty.  Therefore, this division in `calculate_threshold` is safe.
+		let threshold = super::authorship::calculate_primary_threshold(c, authorities, authority_index);
+
+		let pre_digest = get_keypair(&pair)
+			.vrf_sign_after_check(transcript, |inout| super::authorship::check_primary_threshold(inout, threshold))
+			.map(|s| {
+				BabePreDigest::Primary {
+					slot_number,
+					vrf_output: s.0.to_output(),
+					vrf_proof: s.1,
+					authority_index: authority_index as u32,
+				}
+			});
+
+		// early exit on first successful claim
+		if let Some(pre_digest) = pre_digest {
+			return Some((pre_digest, pair));
+		}
+	}
+
+	None
+}

substrate/client/consensus/babe/src/aux_schema.rs

@@ -0,0 +1,103 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Schema for BABE epoch changes in the aux-db.
+
+use log::info;
+use codec::{Decode, Encode};
+
+use client_api::{
+	backend::AuxStore,
+	error::{Result as ClientResult, Error as ClientError},
+};
+use sr_primitives::traits::Block as BlockT;
+use babe_primitives::BabeBlockWeight;
+
+use super::{epoch_changes::EpochChangesFor, SharedEpochChanges};
+
+const BABE_EPOCH_CHANGES: &[u8] = b"babe_epoch_changes";
+
+fn block_weight_key<H: Encode>(block_hash: H) -> Vec<u8> {
+	(b"block_weight", block_hash).encode()
+}
+
+fn load_decode<B, T>(backend: &B, key: &[u8]) -> ClientResult<Option<T>>
+	where
+		B: AuxStore,
+		T: Decode,
+{
+	let corrupt = |e: codec::Error| {
+		ClientError::Backend(format!("BABE DB is corrupted. Decode error: {}", e.what()))
+	};
+	match backend.get_aux(key)? {
+		None => Ok(None),
+		Some(t) => T::decode(&mut &t[..]).map(Some).map_err(corrupt)
+	}
+}
+
+/// Load or initialize persistent epoch change data from backend.
+pub(crate) fn load_epoch_changes<Block: BlockT, B: AuxStore>(
+	backend: &B,
+) -> ClientResult<SharedEpochChanges<Block>> {
+	let epoch_changes = load_decode::<_, EpochChangesFor<Block>>(backend, BABE_EPOCH_CHANGES)?
+		.map(Into::into)
+		.unwrap_or_else(|| {
+			info!(target: "babe",
+				"Creating empty BABE epoch changes on what appears to be first startup."
+			);
+			SharedEpochChanges::new()
+		});
+
+	Ok(epoch_changes)
+}
+
+/// Update the epoch changes on disk after a change.
+pub(crate) fn write_epoch_changes<Block: BlockT, F, R>(
+	epoch_changes: &EpochChangesFor<Block>,
+	write_aux: F,
+) -> R where
+	F: FnOnce(&[(&'static [u8], &[u8])]) -> R,
+{
+	let encoded_epoch_changes = epoch_changes.encode();
+	write_aux(
+		&[(BABE_EPOCH_CHANGES, encoded_epoch_changes.as_slice())],
+	)
+}
+
+/// Write the cumulative chain-weight of a block ot aux storage.
+pub(crate) fn write_block_weight<H: Encode, F, R>(
+	block_hash: H,
+	block_weight: &BabeBlockWeight,
+	write_aux: F,
+) -> R where
+	F: FnOnce(&[(Vec<u8>, &[u8])]) -> R,
+{
+
+	let key = block_weight_key(block_hash);
+	block_weight.using_encoded(|s|
+		write_aux(
+			&[(key, s)],
+		)
+	)
+}
+
+/// Load the cumulative chain-weight associated with a block.
+pub(crate) fn load_block_weight<H: Encode, B: AuxStore>(
+	backend: &B,
+	block_hash: H,
+) -> ClientResult<Option<BabeBlockWeight>> {
+	load_decode(backend, block_weight_key(block_hash).as_slice())
+}

substrate/client/consensus/babe/src/epoch_changes.rs

@@ -0,0 +1,658 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Handling epoch changes in BABE.
+//!
+//! This exposes the `SharedEpochChanges`, which is a wrapper around a
+//! persistent DAG superimposed over the forks of the blockchain.
+
+use std::sync::Arc;
+use babe_primitives::{Epoch, SlotNumber, NextEpochDescriptor};
+use fork_tree::ForkTree;
+use parking_lot::{Mutex, MutexGuard};
+use sr_primitives::traits::{Block as BlockT, NumberFor, One, Zero};
+use codec::{Encode, Decode};
+use client_api::{
+	error::Error as ClientError,
+	utils::is_descendent_of,
+	blockchain::HeaderBackend
+};
+use header_metadata::HeaderMetadata;
+use primitives::H256;
+use std::ops::Add;
+
+/// A builder for `is_descendent_of` functions.
+pub trait IsDescendentOfBuilder<Hash> {
+	/// The error returned by the function.
+	type Error: std::error::Error;
+	/// A function that can tell you if the second parameter is a descendent of
+	/// the first.
+	type IsDescendentOf: Fn(&Hash, &Hash) -> Result<bool, Self::Error>;
+
+	/// Build an `is_descendent_of` function.
+	///
+	/// The `current` parameter can be `Some` with the details a fresh block whose
+	/// details aren't yet stored, but its parent is.
+	///
+	/// The format of `current` when `Some` is `(current, current_parent)`.
+	fn build_is_descendent_of(&self, current: Option<(Hash, Hash)>)
+		-> Self::IsDescendentOf;
+}
+
+/// Produce a descendent query object given the client.
+pub(crate) fn descendent_query<H, Block>(client: &H) -> HeaderBackendDescendentBuilder<&H, Block> {
+	HeaderBackendDescendentBuilder(client, std::marker::PhantomData)
+}
+
+/// Wrapper to get around unconstrained type errors when implementing
+/// `IsDescendentOfBuilder` for header backends.
+pub(crate) struct HeaderBackendDescendentBuilder<H, Block>(H, std::marker::PhantomData<Block>);
+
+// TODO: relying on Hash = H256 is awful.
+// https://github.com/paritytech/substrate/issues/3624
+impl<'a, H, Block> IsDescendentOfBuilder<H256>
+	for HeaderBackendDescendentBuilder<&'a H, Block> where
+	H: HeaderBackend<Block> + HeaderMetadata<Block, Error=ClientError>,
+	Block: BlockT<Hash = H256>,
+{
+	type Error = ClientError;
+	type IsDescendentOf = Box<dyn Fn(&H256, &H256) -> Result<bool, ClientError> + 'a>;
+
+	fn build_is_descendent_of(&self, current: Option<(H256, H256)>)
+		-> Self::IsDescendentOf
+	{
+		Box::new(is_descendent_of(self.0, current))
+	}
+}
+
+/// An unimported genesis epoch.
+pub struct UnimportedGenesis(Epoch);
+
+/// The viable epoch under which a block can be verified.
+///
+/// If this is the first non-genesis block in the chain, then it will
+/// hold an `UnimportedGenesis` epoch.
+pub enum ViableEpoch {
+	Genesis(UnimportedGenesis),
+	Regular(Epoch),
+}
+
+impl From<Epoch> for ViableEpoch {
+	fn from(epoch: Epoch) -> ViableEpoch {
+		ViableEpoch::Regular(epoch)
+	}
+}
+
+impl AsRef<Epoch> for ViableEpoch {
+	fn as_ref(&self) -> &Epoch {
+		match *self {
+			ViableEpoch::Genesis(UnimportedGenesis(ref e)) => e,
+			ViableEpoch::Regular(ref e) => e,
+		}
+	}
+}
+
+impl ViableEpoch {
+	/// Extract the underlying epoch, disregarding the fact that a genesis
+	/// epoch may be unimported.
+	pub fn into_inner(self) -> Epoch {
+		match self {
+			ViableEpoch::Genesis(UnimportedGenesis(e)) => e,
+			ViableEpoch::Regular(e) => e,
+		}
+	}
+
+	/// Increment the epoch, yielding an `IncrementedEpoch` to be imported
+	/// into the fork-tree.
+	pub fn increment(&self, next_descriptor: NextEpochDescriptor) -> IncrementedEpoch {
+		let next = self.as_ref().increment(next_descriptor);
+		let to_persist = match *self {
+			ViableEpoch::Genesis(UnimportedGenesis(ref epoch_0)) =>
+				PersistedEpoch::Genesis(epoch_0.clone(), next),
+			ViableEpoch::Regular(_) => PersistedEpoch::Regular(next),
+		};
+
+		IncrementedEpoch(to_persist)
+	}
+}
+
+/// The datatype encoded on disk.
+// This really shouldn't be public, but the encode/decode derives force it to be.
+#[derive(Clone, Encode, Decode)]
+pub enum PersistedEpoch {
+	// epoch_0, epoch_1,
+	Genesis(Epoch, Epoch),
+	// epoch_n
+	Regular(Epoch),
+}
+
+/// A fresh, incremented epoch to import into the underlying fork-tree.
+///
+/// Create this with `ViableEpoch::increment`.
+#[must_use = "Freshly-incremented epoch must be imported with `EpochChanges::import`"]
+pub struct IncrementedEpoch(PersistedEpoch);
+
+impl AsRef<Epoch> for IncrementedEpoch {
+	fn as_ref(&self) -> &Epoch {
+		match self.0 {
+			PersistedEpoch::Genesis(_, ref epoch_1) => epoch_1,
+			PersistedEpoch::Regular(ref epoch_n) => epoch_n,
+		}
+	}
+}
+
+/// Tree of all epoch changes across all *seen* forks. Data stored in tree is
+/// the hash and block number of the block signaling the epoch change, and the
+/// epoch that was signalled at that block.
+///
+/// BABE special-cases the first epoch, epoch_0, by saying that it starts at
+/// slot number of the first block in the chain. When bootstrapping a chain,
+/// there can be multiple competing block #1s, so we have to ensure that the overlayed
+/// DAG doesn't get confused.
+///
+/// The first block of every epoch should be producing a descriptor for the next
+/// epoch - this is checked in higher-level code. So the first block of epoch_0 contains
+/// a descriptor for epoch_1. We special-case these and bundle them together in the
+/// same DAG entry, pinned to a specific block #1.
+///
+/// Further epochs (epoch_2, ..., epoch_n) each get their own entry.
+#[derive(Clone, Encode, Decode)]
+pub struct EpochChanges<Hash, Number> {
+	inner: ForkTree<Hash, Number, PersistedEpoch>,
+}
+
+// create a fake header hash which hasn't been included in the chain.
+fn fake_head_hash<H: AsRef<[u8]> + AsMut<[u8]> + Clone>(parent_hash: &H) -> H {
+	let mut h = parent_hash.clone();
+	// dirty trick: flip the first bit of the parent hash to create a hash
+	// which has not been in the chain before (assuming a strong hash function).
+	h.as_mut()[0] ^= 0b10000000;
+	h
+}
+
+impl<Hash, Number> EpochChanges<Hash, Number> where
+	Hash: PartialEq + AsRef<[u8]> + AsMut<[u8]> + Copy,
+	Number: Ord + One + Zero + Add<Output=Number> + Copy,
+{
+	/// Create a new epoch-change tracker.
+	fn new() -> Self {
+		EpochChanges { inner: ForkTree::new() }
+	}
+
+	/// Prune out finalized epochs, except for the ancestor of the finalized
+	/// block. The given slot should be the slot number at which the finalized
+	/// block was authored.
+	pub fn prune_finalized<D: IsDescendentOfBuilder<Hash>>(
+		&mut self,
+		descendent_of_builder: D,
+		hash: &Hash,
+		number: Number,
+		slot: SlotNumber,
+	) -> Result<(), fork_tree::Error<D::Error>> {
+		let is_descendent_of = descendent_of_builder
+			.build_is_descendent_of(None);
+
+		let predicate = |epoch: &PersistedEpoch| match *epoch {
+			PersistedEpoch::Genesis(_, ref epoch_1) =>
+				slot >= epoch_1.end_slot(),
+			PersistedEpoch::Regular(ref epoch_n) =>
+				slot >= epoch_n.end_slot(),
+		};
+
+		// prune any epochs which could not be _live_ as of the children of the
+		// finalized block, i.e. re-root the fork tree to the oldest ancestor of
+		// (hash, number) where epoch.end_slot() >= finalized_slot
+		self.inner.prune(
+			hash,
+			&number,
+			&is_descendent_of,
+			&predicate,
+		)?;
+
+		Ok(())
+	}
+
+	/// Finds the epoch for a child of the given block, assuming the given slot number.
+	///
+	/// If the returned epoch is an `UnimportedGenesis` epoch, it should be imported into the
+	/// tree.
+	pub fn epoch_for_child_of<D: IsDescendentOfBuilder<Hash>, G>(
+		&self,
+		descendent_of_builder: D,
+		parent_hash: &Hash,
+		parent_number: Number,
+		slot_number: SlotNumber,
+		make_genesis: G,
+	) -> Result<Option<ViableEpoch>, fork_tree::Error<D::Error>>
+		where G: FnOnce(SlotNumber) -> Epoch
+	{
+		// find_node_where will give you the node in the fork-tree which is an ancestor
+		// of the `parent_hash` by default. if the last epoch was signalled at the parent_hash,
+		// then it won't be returned. we need to create a new fake chain head hash which
+		// "descends" from our parent-hash.
+		let fake_head_hash = fake_head_hash(parent_hash);
+
+		let is_descendent_of = descendent_of_builder
+			.build_is_descendent_of(Some((fake_head_hash, *parent_hash)));
+
+		if parent_number == Zero::zero() {
+			// need to insert the genesis epoch.
+			let genesis_epoch = make_genesis(slot_number);
+			return Ok(Some(ViableEpoch::Genesis(UnimportedGenesis(genesis_epoch))));
+		}
+
+		// We want to find the deepest node in the tree which is an ancestor
+		// of our block and where the start slot of the epoch was before the
+		// slot of our block. The genesis special-case doesn't need to look
+		// at epoch_1 -- all we're doing here is figuring out which node
+		// we need.
+		let predicate = |epoch: &PersistedEpoch| match *epoch {
+			PersistedEpoch::Genesis(ref epoch_0, _) =>
+				epoch_0.start_slot <= slot_number,
+			PersistedEpoch::Regular(ref epoch_n) =>
+				epoch_n.start_slot <= slot_number,
+		};
+
+		self.inner.find_node_where(
+			&fake_head_hash,
+			&(parent_number + One::one()),
+			&is_descendent_of,
+			&predicate,
+		)
+			.map(|n| n.map(|node| ViableEpoch::Regular(match node.data {
+				// Ok, we found our node.
+				// and here we figure out which of the internal epochs
+				// of a genesis node to use based on their start slot.
+				PersistedEpoch::Genesis(ref epoch_0, ref epoch_1) =>
+					if epoch_1.start_slot <= slot_number {
+						epoch_1.clone()
+					} else {
+						epoch_0.clone()
+					},
+				PersistedEpoch::Regular(ref epoch_n) => epoch_n.clone(),
+			})))
+	}
+
+	/// Import a new epoch-change, signalled at the given block.
+	///
+	/// This assumes that the given block is prospective (i.e. has not been
+	/// imported yet), but its parent has. This is why the parent hash needs
+	/// to be provided.
+	pub fn import<D: IsDescendentOfBuilder<Hash>>(
+		&mut self,
+		descendent_of_builder: D,
+		hash: Hash,
+		number: Number,
+		parent_hash: Hash,
+		epoch: IncrementedEpoch,
+	) -> Result<(), fork_tree::Error<D::Error>> {
+		let is_descendent_of = descendent_of_builder
+			.build_is_descendent_of(Some((hash, parent_hash)));
+
+		let res = self.inner.import(
+			hash,
+			number,
+			epoch.0,
+			&is_descendent_of,
+		);
+
+		match res {
+			Ok(_) | Err(fork_tree::Error::Duplicate) => Ok(()),
+			Err(e) => Err(e),
+		}
+	}
+
+	/// Return the inner fork tree, useful for testing purposes.
+	#[cfg(test)]
+	pub fn tree(&self) -> &ForkTree<Hash, Number, PersistedEpoch> {
+		&self.inner
+	}
+}
+
+/// Type alias to produce the epoch-changes tree from a block type.
+pub type EpochChangesFor<Block> = EpochChanges<<Block as BlockT>::Hash, NumberFor<Block>>;
+
+/// A shared epoch changes tree.
+#[derive(Clone)]
+pub struct SharedEpochChanges<Block: BlockT> {
+	inner: Arc<Mutex<EpochChangesFor<Block>>>,
+}
+
+impl<Block: BlockT> SharedEpochChanges<Block> {
+	/// Create a new instance of the `SharedEpochChanges`.
+	pub fn new() -> Self {
+		SharedEpochChanges {
+			inner: Arc::new(Mutex::new(EpochChanges::<_, _>::new()))
+		}
+	}
+
+	/// Lock the shared epoch changes,
+	pub fn lock(&self) -> MutexGuard<EpochChangesFor<Block>> {
+		self.inner.lock()
+	}
+}
+
+impl<Block: BlockT> From<EpochChangesFor<Block>> for SharedEpochChanges<Block> {
+	fn from(epoch_changes: EpochChangesFor<Block>) -> Self {
+		SharedEpochChanges {
+			inner: Arc::new(Mutex::new(epoch_changes))
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[derive(Debug, PartialEq)]
+	pub struct TestError;
+
+	impl std::fmt::Display for TestError {
+		fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+			write!(f, "TestError")
+		}
+	}
+
+	impl std::error::Error for TestError {}
+
+	impl<'a, F: 'a , H: 'a + PartialEq + std::fmt::Debug> IsDescendentOfBuilder<H> for &'a F
+		where F: Fn(&H, &H) -> Result<bool, TestError>
+	{
+		type Error = TestError;
+		type IsDescendentOf = Box<dyn Fn(&H, &H) -> Result<bool, TestError> + 'a>;
+
+		fn build_is_descendent_of(&self, current: Option<(H, H)>)
+			-> Self::IsDescendentOf
+		{
+			let f = *self;
+			Box::new(move |base, head| {
+				let mut head = head;
+
+				if let Some((ref c_head, ref c_parent)) = current {
+					if head == c_head {
+						if base == c_parent {
+							return Ok(true);
+						} else {
+							head = c_parent;
+						}
+					}
+				}
+
+				f(base, head)
+			})
+		}
+	}
+
+	type Hash = [u8; 1];
+
+	#[test]
+	fn genesis_epoch_is_created_but_not_imported() {
+		//
+		// A - B
+		//  \
+		//   — C
+		//
+		let is_descendent_of = |base: &Hash, block: &Hash| -> Result<bool, TestError> {
+			match (base, *block) {
+				(b"A", b) => Ok(b == *b"B" || b == *b"C" || b == *b"D"),
+				(b"B", b) | (b"C", b) => Ok(b == *b"D"),
+				(b"0", _) => Ok(true),
+				_ => Ok(false),
+			}
+		};
+
+		let make_genesis = |slot| Epoch {
+			epoch_index: 0,
+			start_slot: slot,
+			duration: 100,
+			authorities: Vec::new(),
+			randomness: [0; 32],
+		};
+
+		let epoch_changes = EpochChanges::new();
+		let genesis_epoch = epoch_changes.epoch_for_child_of(
+			&is_descendent_of,
+			b"0",
+			0,
+			10101,
+			&make_genesis,
+		).unwrap().unwrap();
+
+		match genesis_epoch {
+			ViableEpoch::Genesis(_) => {},
+			_ => panic!("should be unimported genesis"),
+		};
+		assert_eq!(genesis_epoch.as_ref(), &make_genesis(10101));
+
+		let genesis_epoch_2 = epoch_changes.epoch_for_child_of(
+			&is_descendent_of,
+			b"0",
+			0,
+			10102,
+			&make_genesis,
+		).unwrap().unwrap();
+
+		match genesis_epoch_2 {
+			ViableEpoch::Genesis(_) => {},
+			_ => panic!("should be unimported genesis"),
+		};
+		assert_eq!(genesis_epoch_2.as_ref(), &make_genesis(10102));
+	}
+
+	#[test]
+	fn epoch_changes_between_blocks() {
+		//
+		// A - B
+		//  \
+		//   — C
+		//
+		let is_descendent_of = |base: &Hash, block: &Hash| -> Result<bool, TestError> {
+			match (base, *block) {
+				(b"A", b) => Ok(b == *b"B" || b == *b"C" || b == *b"D"),
+				(b"B", b) | (b"C", b) => Ok(b == *b"D"),
+				(b"0", _) => Ok(true),
+				_ => Ok(false),
+			}
+		};
+
+		let make_genesis = |slot| Epoch {
+			epoch_index: 0,
+			start_slot: slot,
+			duration: 100,
+			authorities: Vec::new(),
+			randomness: [0; 32],
+		};
+
+		let mut epoch_changes = EpochChanges::new();
+		let genesis_epoch = epoch_changes.epoch_for_child_of(
+			&is_descendent_of,
+			b"0",
+			0,
+			100,
+			&make_genesis,
+		).unwrap().unwrap();
+
+		assert_eq!(genesis_epoch.as_ref(), &make_genesis(100));
+
+		let import_epoch_1 = genesis_epoch.increment(NextEpochDescriptor {
+			authorities: Vec::new(),
+			randomness: [1; 32],
+		});
+		let epoch_1 = import_epoch_1.as_ref().clone();
+
+		epoch_changes.import(
+			&is_descendent_of,
+			*b"A",
+			1,
+			*b"0",
+			import_epoch_1,
+		).unwrap();
+		let genesis_epoch = genesis_epoch.into_inner();
+
+		assert!(is_descendent_of(b"0", b"A").unwrap());
+
+		let end_slot = genesis_epoch.end_slot();
+		assert_eq!(end_slot, epoch_1.start_slot);
+
+		{
+			// x is still within the genesis epoch.
+			let x = epoch_changes.epoch_for_child_of(
+				&is_descendent_of,
+				b"A",
+				1,
+				end_slot - 1,
+				&make_genesis,
+			).unwrap().unwrap().into_inner();
+
+			assert_eq!(x, genesis_epoch);
+		}
+
+		{
+			// x is now at the next epoch, because the block is now at the
+			// start slot of epoch 1.
+			let x = epoch_changes.epoch_for_child_of(
+				&is_descendent_of,
+				b"A",
+				1,
+				end_slot,
+				&make_genesis,
+			).unwrap().unwrap().into_inner();
+
+			assert_eq!(x, epoch_1);
+		}
+
+		{
+			// x is now at the next epoch, because the block is now after
+			// start slot of epoch 1.
+			let x = epoch_changes.epoch_for_child_of(
+				&is_descendent_of,
+				b"A",
+				1,
+				epoch_1.end_slot() - 1,
+				&make_genesis,
+			).unwrap().unwrap().into_inner();
+
+			assert_eq!(x, epoch_1);
+		}
+	}
+
+	#[test]
+	fn two_block_ones_dont_conflict() {
+		//     X - Y
+		//   /
+		// 0 - A - B
+		//
+		let is_descendent_of = |base: &Hash, block: &Hash| -> Result<bool, TestError> {
+			match (base, *block) {
+				(b"A", b) => Ok(b == *b"B"),
+				(b"X", b) => Ok(b == *b"Y"),
+				(b"0", _) => Ok(true),
+				_ => Ok(false),
+			}
+		};
+
+		let duration = 100;
+
+		let make_genesis = |slot| Epoch {
+			epoch_index: 0,
+			start_slot: slot,
+			duration,
+			authorities: Vec::new(),
+			randomness: [0; 32],
+		};
+
+		let mut epoch_changes = EpochChanges::new();
+		let next_descriptor = NextEpochDescriptor {
+			authorities: Vec::new(),
+			randomness: [0; 32],
+		};
+
+		// insert genesis epoch for A
+		{
+			let genesis_epoch_a = epoch_changes.epoch_for_child_of(
+				&is_descendent_of,
+				b"0",
+				0,
+				100,
+				&make_genesis,
+			).unwrap().unwrap();
+
+			epoch_changes.import(
+				&is_descendent_of,
+				*b"A",
+				1,
+				*b"0",
+				genesis_epoch_a.increment(next_descriptor.clone()),
+			).unwrap();
+
+		}
+
+		// insert genesis epoch for X
+		{
+			let genesis_epoch_x = epoch_changes.epoch_for_child_of(
+				&is_descendent_of,
+				b"0",
+				0,
+				1000,
+				&make_genesis,
+			).unwrap().unwrap();
+
+			epoch_changes.import(
+				&is_descendent_of,
+				*b"X",
+				1,
+				*b"0",
+				genesis_epoch_x.increment(next_descriptor.clone()),
+			).unwrap();
+		}
+
+		// now check that the genesis epochs for our respective block 1s
+		// respect the chain structure.
+		{
+			let epoch_for_a_child = epoch_changes.epoch_for_child_of(
+				&is_descendent_of,
+				b"A",
+				1,
+				101,
+				&make_genesis,
+			).unwrap().unwrap();
+
+			assert_eq!(epoch_for_a_child.into_inner(), make_genesis(100));
+
+			let epoch_for_x_child = epoch_changes.epoch_for_child_of(
+				&is_descendent_of,
+				b"X",
+				1,
+				1001,
+				&make_genesis,
+			).unwrap().unwrap();
+
+			assert_eq!(epoch_for_x_child.into_inner(), make_genesis(1000));
+
+			let epoch_for_x_child_before_genesis = epoch_changes.epoch_for_child_of(
+				&is_descendent_of,
+				b"X",
+				1,
+				101,
+				&make_genesis,
+			).unwrap();
+
+			// even though there is a genesis epoch at that slot, it's not in
+			// this chain.
+			assert!(epoch_for_x_child_before_genesis.is_none());
+		}
+	}
+}

substrate/client/consensus/babe/src/tests.rs

@@ -0,0 +1,779 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! BABE testsuite
+
+// FIXME #2532: need to allow deprecated until refactor is done
+// https://github.com/paritytech/substrate/issues/2532
+#![allow(deprecated)]
+use super::*;
+use authorship::claim_slot;
+
+use babe_primitives::{AuthorityPair, SlotNumber};
+use block_builder::BlockBuilder;
+use consensus_common::NoNetwork as DummyOracle;
+use consensus_common::import_queue::{
+	BoxBlockImport, BoxJustificationImport, BoxFinalityProofImport,
+};
+use network::test::*;
+use network::test::{Block as TestBlock, PeersClient};
+use network::config::BoxFinalityProofRequestBuilder;
+use sr_primitives::{generic::DigestItem, traits::{Block as BlockT, DigestFor}};
+use network::config::ProtocolConfig;
+use tokio::runtime::current_thread;
+use client_api::BlockchainEvents;
+use test_client;
+use log::debug;
+use std::{time::Duration, cell::RefCell};
+
+type Item = DigestItem<Hash>;
+
+type Error = client::error::Error;
+
+type TestClient = client::Client<
+	test_client::Backend,
+	test_client::Executor,
+	TestBlock,
+	test_client::runtime::RuntimeApi,
+>;
+
+#[derive(Copy, Clone, PartialEq)]
+enum Stage {
+	PreSeal,
+	PostSeal,
+}
+
+type Mutator = Arc<dyn Fn(&mut TestHeader, Stage) + Send + Sync>;
+
+#[derive(Clone)]
+struct DummyFactory {
+	client: Arc<TestClient>,
+	epoch_changes: crate::SharedEpochChanges<TestBlock>,
+	config: Config,
+	mutator: Mutator,
+}
+
+struct DummyProposer {
+	factory: DummyFactory,
+	parent_hash: Hash,
+	parent_number: u64,
+	parent_slot: SlotNumber,
+}
+
+impl Environment<TestBlock> for DummyFactory {
+	type Proposer = DummyProposer;
+	type Error = Error;
+
+	fn init(&mut self, parent_header: &<TestBlock as BlockT>::Header)
+		-> Result<DummyProposer, Error>
+	{
+
+		let parent_slot = crate::find_pre_digest::<TestBlock>(parent_header)
+			.expect("parent header has a pre-digest")
+			.slot_number();
+
+		Ok(DummyProposer {
+			factory: self.clone(),
+			parent_hash: parent_header.hash(),
+			parent_number: *parent_header.number(),
+			parent_slot,
+		})
+	}
+}
+
+impl DummyProposer {
+	fn propose_with(&mut self, pre_digests: DigestFor<TestBlock>)
+		-> future::Ready<Result<TestBlock, Error>>
+	{
+		use codec::Encode;
+		let block_builder = self.factory.client.new_block_at(
+			&BlockId::Hash(self.parent_hash),
+			pre_digests,
+		).unwrap();
+
+		let mut block = match block_builder.bake().map_err(|e| e.into()) {
+			Ok(b) => b,
+			Err(e) => return future::ready(Err(e)),
+		};
+
+		let this_slot = crate::find_pre_digest::<TestBlock>(block.header())
+			.expect("baked block has valid pre-digest")
+			.slot_number();
+
+		// figure out if we should add a consensus digest, since the test runtime
+		// doesn't.
+		let epoch_changes = self.factory.epoch_changes.lock();
+		let epoch = epoch_changes.epoch_for_child_of(
+			descendent_query(&*self.factory.client),
+			&self.parent_hash,
+			self.parent_number,
+			this_slot,
+			|slot| self.factory.config.genesis_epoch(slot),
+		)
+			.expect("client has data to find epoch")
+			.expect("can compute epoch for baked block")
+			.into_inner();
+
+		let first_in_epoch = self.parent_slot < epoch.start_slot;
+		if first_in_epoch {
+			// push a `Consensus` digest signalling next change.
+			// we just reuse the same randomness and authorities as the prior
+			// epoch. this will break when we add light client support, since
+			// that will re-check the randomness logic off-chain.
+			let digest_data = ConsensusLog::NextEpochData(NextEpochDescriptor {
+				authorities: epoch.authorities.clone(),
+				randomness: epoch.randomness.clone(),
+			}).encode();
+			let digest = DigestItem::Consensus(BABE_ENGINE_ID, digest_data);
+			block.header.digest_mut().push(digest)
+		}
+
+		// mutate the block header according to the mutator.
+		(self.factory.mutator)(&mut block.header, Stage::PreSeal);
+
+		future::ready(Ok(block))
+	}
+}
+
+impl Proposer<TestBlock> for DummyProposer {
+	type Error = Error;
+	type Create = future::Ready<Result<TestBlock, Error>>;
+
+	fn propose(
+		&mut self,
+		_: InherentData,
+		pre_digests: DigestFor<TestBlock>,
+		_: Duration,
+	) -> Self::Create {
+		self.propose_with(pre_digests)
+	}
+}
+
+thread_local! {
+	static MUTATOR: RefCell<Mutator> = RefCell::new(Arc::new(|_, _|()));
+}
+
+#[derive(Clone)]
+struct PanickingBlockImport<B>(B);
+
+impl<B: BlockImport<TestBlock>> BlockImport<TestBlock> for PanickingBlockImport<B> {
+	type Error = B::Error;
+
+	fn import_block(
+		&mut self,
+		block: BlockImportParams<TestBlock>,
+		new_cache: HashMap<CacheKeyId, Vec<u8>>,
+	) -> Result<ImportResult, Self::Error> {
+		Ok(self.0.import_block(block, new_cache).expect("importing block failed"))
+	}
+
+	fn check_block(
+		&mut self,
+		block: BlockCheckParams<TestBlock>,
+	) -> Result<ImportResult, Self::Error> {
+		Ok(self.0.check_block(block).expect("checking block failed"))
+	}
+}
+
+pub struct BabeTestNet {
+	peers: Vec<Peer<Option<PeerData>, DummySpecialization>>,
+}
+
+type TestHeader = <TestBlock as BlockT>::Header;
+type TestExtrinsic = <TestBlock as BlockT>::Extrinsic;
+
+pub struct TestVerifier {
+	inner: BabeVerifier<
+		test_client::Backend,
+		test_client::Executor,
+		TestBlock,
+		test_client::runtime::RuntimeApi,
+		PeersFullClient,
+	>,
+	mutator: Mutator,
+}
+
+impl Verifier<TestBlock> for TestVerifier {
+	/// Verify the given data and return the BlockImportParams and an optional
+	/// new set of validators to import. If not, err with an Error-Message
+	/// presented to the User in the logs.
+	fn verify(
+		&mut self,
+		origin: BlockOrigin,
+		mut header: TestHeader,
+		justification: Option<Justification>,
+		body: Option<Vec<TestExtrinsic>>,
+	) -> Result<(BlockImportParams<TestBlock>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
+		// apply post-sealing mutations (i.e. stripping seal, if desired).
+		(self.mutator)(&mut header, Stage::PostSeal);
+		Ok(self.inner.verify(origin, header, justification, body).expect("verification failed!"))
+	}
+}
+
+pub struct PeerData {
+	link: BabeLink<TestBlock>,
+	inherent_data_providers: InherentDataProviders,
+	block_import: Mutex<Option<BoxBlockImport<TestBlock>>>,
+}
+
+impl TestNetFactory for BabeTestNet {
+	type Specialization = DummySpecialization;
+	type Verifier = TestVerifier;
+	type PeerData = Option<PeerData>;
+
+	/// Create new test network with peers and given config.
+	fn from_config(_config: &ProtocolConfig) -> Self {
+		debug!(target: "babe", "Creating test network from config");
+		BabeTestNet {
+			peers: Vec::new(),
+		}
+	}
+
+	fn make_block_import(&self, client: PeersClient)
+		-> (
+			BoxBlockImport<Block>,
+			Option<BoxJustificationImport<Block>>,
+			Option<BoxFinalityProofImport<Block>>,
+			Option<BoxFinalityProofRequestBuilder<Block>>,
+			Option<PeerData>,
+		)
+	{
+		let client = client.as_full().expect("only full clients are tested");
+		let inherent_data_providers = InherentDataProviders::new();
+
+		let config = Config::get_or_compute(&*client).expect("config available");
+		let (block_import, link) = crate::block_import(
+			config,
+			client.clone(),
+			client.clone(),
+			client.clone(),
+		).expect("can initialize block-import");
+
+		let block_import = PanickingBlockImport(block_import);
+
+		let data_block_import = Mutex::new(Some(Box::new(block_import.clone()) as BoxBlockImport<_>));
+		(
+			Box::new(block_import),
+			None,
+			None,
+			None,
+			Some(PeerData { link, inherent_data_providers, block_import: data_block_import }),
+		)
+	}
+
+	fn make_verifier(
+		&self,
+		client: PeersClient,
+		_cfg: &ProtocolConfig,
+		maybe_link: &Option<PeerData>,
+	)
+		-> Self::Verifier
+	{
+		let client = client.as_full().expect("only full clients are used in test");
+		trace!(target: "babe", "Creating a verifier");
+
+		// ensure block import and verifier are linked correctly.
+		let data = maybe_link.as_ref().expect("babe link always provided to verifier instantiation");
+
+		TestVerifier {
+			inner: BabeVerifier {
+				client: client.clone(),
+				api: client,
+				inherent_data_providers: data.inherent_data_providers.clone(),
+				config: data.link.config.clone(),
+				epoch_changes: data.link.epoch_changes.clone(),
+				time_source: data.link.time_source.clone(),
+			},
+			mutator: MUTATOR.with(|m| m.borrow().clone()),
+		}
+	}
+
+	fn peer(&mut self, i: usize) -> &mut Peer<Self::PeerData, DummySpecialization> {
+		trace!(target: "babe", "Retreiving a peer");
+		&mut self.peers[i]
+	}
+
+	fn peers(&self) -> &Vec<Peer<Self::PeerData, DummySpecialization>> {
+		trace!(target: "babe", "Retreiving peers");
+		&self.peers
+	}
+
+	fn mut_peers<F: FnOnce(&mut Vec<Peer<Self::PeerData, DummySpecialization>>)>(
+		&mut self,
+		closure: F,
+	) {
+		closure(&mut self.peers);
+	}
+}
+
+#[test]
+#[should_panic]
+fn rejects_empty_block() {
+	env_logger::try_init().unwrap();
+	let mut net = BabeTestNet::new(3);
+	let block_builder = |builder: BlockBuilder<_, _>| {
+		builder.bake().unwrap()
+	};
+	net.mut_peers(|peer| {
+		peer[0].generate_blocks(1, BlockOrigin::NetworkInitialSync, block_builder);
+	})
+}
+
+fn run_one_test(
+	mutator: impl Fn(&mut TestHeader, Stage) + Send + Sync + 'static,
+) {
+	let _ = env_logger::try_init();
+	let mutator = Arc::new(mutator) as Mutator;
+
+	MUTATOR.with(|m| *m.borrow_mut() = mutator.clone());
+	let net = BabeTestNet::new(3);
+
+	let peers = &[
+		(0, "//Alice"),
+		(1, "//Bob"),
+		(2, "//Charlie"),
+	];
+
+	let net = Arc::new(Mutex::new(net));
+	let mut import_notifications = Vec::new();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut keystore_paths = Vec::new();
+
+	for (peer_id, seed) in peers {
+		let mut net = net.lock();
+		let peer = net.peer(*peer_id);
+		let client = peer.client().as_full().expect("Only full clients are used in tests").clone();
+		let select_chain = peer.select_chain().expect("Full client has select_chain");
+
+		let keystore_path = tempfile::tempdir().expect("Creates keystore path");
+		let keystore = keystore::Store::open(keystore_path.path(), None).expect("Creates keystore");
+		keystore.write().insert_ephemeral_from_seed::<AuthorityPair>(seed).expect("Generates authority key");
+		keystore_paths.push(keystore_path);
+
+		let mut got_own = false;
+		let mut got_other = false;
+
+		let data = peer.data.as_ref().expect("babe link set up during initialization");
+
+		let environ = DummyFactory {
+			client: client.clone(),
+			config: data.link.config.clone(),
+			epoch_changes: data.link.epoch_changes.clone(),
+			mutator: mutator.clone(),
+		};
+
+		import_notifications.push(
+			// run each future until we get one of our own blocks with number higher than 5
+			// that was produced locally.
+			client.import_notification_stream()
+				.take_while(move |n| future::ready(n.header.number() < &5 || {
+					if n.origin == BlockOrigin::Own {
+						got_own = true;
+					} else {
+						got_other = true;
+					}
+
+					// continue until we have at least one block of our own
+					// and one of another peer.
+					!(got_own && got_other)
+				}))
+				.for_each(|_| future::ready(()) )
+		);
+
+
+		runtime.spawn(start_babe(BabeParams {
+			block_import: data.block_import.lock().take().expect("import set up during init"),
+			select_chain,
+			client,
+			env: environ,
+			sync_oracle: DummyOracle,
+			inherent_data_providers: data.inherent_data_providers.clone(),
+			force_authoring: false,
+			babe_link: data.link.clone(),
+			keystore,
+		}).expect("Starts babe"));
+	}
+
+	runtime.spawn(futures01::future::poll_fn(move || {
+		net.lock().poll();
+		Ok::<_, ()>(futures01::Async::NotReady::<()>)
+	}));
+
+	runtime.block_on(future::join_all(import_notifications)
+		.map(|_| Ok::<(), ()>(())).compat()).unwrap();
+}
+
+#[test]
+fn authoring_blocks() {
+	run_one_test(|_, _| ())
+}
+
+#[test]
+#[should_panic]
+fn rejects_missing_inherent_digest() {
+	run_one_test(|header: &mut TestHeader, stage| {
+		let v = std::mem::replace(&mut header.digest_mut().logs, vec![]);
+		header.digest_mut().logs = v.into_iter()
+			.filter(|v| stage == Stage::PostSeal || v.as_babe_pre_digest().is_none())
+			.collect()
+	})
+}
+
+#[test]
+#[should_panic]
+fn rejects_missing_seals() {
+	run_one_test(|header: &mut TestHeader, stage| {
+		let v = std::mem::replace(&mut header.digest_mut().logs, vec![]);
+		header.digest_mut().logs = v.into_iter()
+			.filter(|v| stage == Stage::PreSeal || v.as_babe_seal().is_none())
+			.collect()
+	})
+}
+
+#[test]
+#[should_panic]
+fn rejects_missing_consensus_digests() {
+	run_one_test(|header: &mut TestHeader, stage| {
+		let v = std::mem::replace(&mut header.digest_mut().logs, vec![]);
+		header.digest_mut().logs = v.into_iter()
+			.filter(|v| stage == Stage::PostSeal || v.as_next_epoch_descriptor().is_none())
+			.collect()
+	});
+}
+
+#[test]
+fn wrong_consensus_engine_id_rejected() {
+	let _ = env_logger::try_init();
+	let sig = AuthorityPair::generate().0.sign(b"");
+	let bad_seal: Item = DigestItem::Seal([0; 4], sig.to_vec());
+	assert!(bad_seal.as_babe_pre_digest().is_none());
+	assert!(bad_seal.as_babe_seal().is_none())
+}
+
+#[test]
+fn malformed_pre_digest_rejected() {
+	let _ = env_logger::try_init();
+	let bad_seal: Item = DigestItem::Seal(BABE_ENGINE_ID, [0; 64].to_vec());
+	assert!(bad_seal.as_babe_pre_digest().is_none());
+}
+
+#[test]
+fn sig_is_not_pre_digest() {
+	let _ = env_logger::try_init();
+	let sig = AuthorityPair::generate().0.sign(b"");
+	let bad_seal: Item = DigestItem::Seal(BABE_ENGINE_ID, sig.to_vec());
+	assert!(bad_seal.as_babe_pre_digest().is_none());
+	assert!(bad_seal.as_babe_seal().is_some())
+}
+
+#[test]
+fn can_author_block() {
+	let _ = env_logger::try_init();
+	let keystore_path = tempfile::tempdir().expect("Creates keystore path");
+	let keystore = keystore::Store::open(keystore_path.path(), None).expect("Creates keystore");
+	let pair = keystore.write().insert_ephemeral_from_seed::<AuthorityPair>("//Alice")
+		.expect("Generates authority pair");
+
+	let mut i = 0;
+	let epoch = Epoch {
+		start_slot: 0,
+		authorities: vec![(pair.public(), 1)],
+		randomness: [0; 32],
+		epoch_index: 1,
+		duration: 100,
+	};
+
+	let mut config = crate::BabeConfiguration {
+		slot_duration: 1000,
+		epoch_length: 100,
+		c: (3, 10),
+		genesis_authorities: Vec::new(),
+		randomness: [0; 32],
+		secondary_slots: true,
+	};
+
+	// with secondary slots enabled it should never be empty
+	match claim_slot(i, &epoch, &config, &keystore) {
+		None => i += 1,
+		Some(s) => debug!(target: "babe", "Authored block {:?}", s.0),
+	}
+
+	// otherwise with only vrf-based primary slots we might need to try a couple
+	// of times.
+	config.secondary_slots = false;
+	loop {
+		match claim_slot(i, &epoch, &config, &keystore) {
+			None => i += 1,
+			Some(s) => {
+				debug!(target: "babe", "Authored block {:?}", s.0);
+				break;
+			}
+		}
+	}
+}
+
+// Propose and import a new BABE block on top of the given parent.
+fn propose_and_import_block(
+	parent: &TestHeader,
+	slot_number: Option<SlotNumber>,
+	proposer_factory: &mut DummyFactory,
+	block_import: &mut BoxBlockImport<TestBlock>,
+) -> H256 {
+	let mut proposer = proposer_factory.init(parent).unwrap();
+
+	let slot_number = slot_number.unwrap_or_else(|| {
+		let parent_pre_digest = find_pre_digest::<TestBlock>(parent).unwrap();
+		parent_pre_digest.slot_number() + 1
+	});
+
+	let pre_digest = sr_primitives::generic::Digest {
+		logs: vec![
+			Item::babe_pre_digest(
+				BabePreDigest::Secondary {
+					authority_index: 0,
+					slot_number,
+				},
+			),
+		],
+	};
+
+	let mut block = futures::executor::block_on(proposer.propose_with(pre_digest)).unwrap();
+
+	let seal = {
+		// sign the pre-sealed hash of the block and then
+		// add it to a digest item.
+		let pair = AuthorityPair::from_seed(&[1; 32]);
+		let pre_hash = block.header.hash();
+		let signature = pair.sign(pre_hash.as_ref());
+		Item::babe_seal(signature)
+	};
+
+	let post_hash = {
+		block.header.digest_mut().push(seal.clone());
+		let h = block.header.hash();
+		block.header.digest_mut().pop();
+		h
+	};
+
+	let import_result = block_import.import_block(
+		BlockImportParams {
+			origin: BlockOrigin::Own,
+			header: block.header,
+			justification: None,
+			post_digests: vec![seal],
+			body: Some(block.extrinsics),
+			finalized: false,
+			auxiliary: Vec::new(),
+			fork_choice: ForkChoiceStrategy::LongestChain,
+			allow_missing_state: false,
+		},
+		Default::default(),
+	).unwrap();
+
+	match import_result {
+		ImportResult::Imported(_) => {},
+		_ => panic!("expected block to be imported"),
+	}
+
+	post_hash
+}
+
+#[test]
+fn importing_block_one_sets_genesis_epoch() {
+	let mut net = BabeTestNet::new(1);
+
+	let peer = net.peer(0);
+	let data = peer.data.as_ref().expect("babe link set up during initialization");
+	let client = peer.client().as_full().expect("Only full clients are used in tests").clone();
+
+	let mut proposer_factory = DummyFactory {
+		client: client.clone(),
+		config: data.link.config.clone(),
+		epoch_changes: data.link.epoch_changes.clone(),
+		mutator: Arc::new(|_, _| ()),
+	};
+
+	let mut block_import = data.block_import.lock().take().expect("import set up during init");
+
+	let genesis_header = client.header(&BlockId::Number(0)).unwrap().unwrap();
+
+	let block_hash = propose_and_import_block(
+		&genesis_header,
+		Some(999),
+		&mut proposer_factory,
+		&mut block_import,
+	);
+
+	let genesis_epoch = data.link.config.genesis_epoch(999);
+
+	let epoch_changes = data.link.epoch_changes.lock();
+	let epoch_for_second_block = epoch_changes.epoch_for_child_of(
+		descendent_query(&*client),
+		&block_hash,
+		1,
+		1000,
+		|slot| data.link.config.genesis_epoch(slot),
+	).unwrap().unwrap().into_inner();
+
+	assert_eq!(epoch_for_second_block, genesis_epoch);
+}
+
+#[test]
+fn importing_epoch_change_block_prunes_tree() {
+	use client_api::Finalizer;
+
+	let mut net = BabeTestNet::new(1);
+
+	let peer = net.peer(0);
+	let data = peer.data.as_ref().expect("babe link set up during initialization");
+
+	let client = peer.client().as_full().expect("Only full clients are used in tests").clone();
+	let mut block_import = data.block_import.lock().take().expect("import set up during init");
+	let epoch_changes = data.link.epoch_changes.clone();
+
+	let mut proposer_factory = DummyFactory {
+		client: client.clone(),
+		config: data.link.config.clone(),
+		epoch_changes: data.link.epoch_changes.clone(),
+		mutator: Arc::new(|_, _| ()),
+	};
+
+	// This is just boilerplate code for proposing and importing n valid BABE
+	// blocks that are built on top of the given parent. The proposer takes care
+	// of producing epoch change digests according to the epoch duration (which
+	// is set to 6 slots in the test runtime).
+	let mut propose_and_import_blocks = |parent_id, n| {
+		let mut hashes = Vec::new();
+		let mut parent_header = client.header(&parent_id).unwrap().unwrap();
+
+		for _ in 0..n {
+			let block_hash = propose_and_import_block(
+				&parent_header,
+				None,
+				&mut proposer_factory,
+				&mut block_import,
+			);
+			hashes.push(block_hash);
+			parent_header = client.header(&BlockId::Hash(block_hash)).unwrap().unwrap();
+		}
+
+		hashes
+	};
+
+	// This is the block tree that we're going to use in this test. Each node
+	// represents an epoch change block, the epoch duration is 6 slots.
+	//
+	//    *---- F (#7)
+	//   /                 *------ G (#19) - H (#25)
+	//  /                 /
+	// A (#1) - B (#7) - C (#13) - D (#19) - E (#25)
+	//                              \
+	//                               *------ I (#25)
+
+	// Create and import the canon chain and keep track of fork blocks (A, C, D)
+	// from the diagram above.
+	let canon_hashes = propose_and_import_blocks(BlockId::Number(0), 30);
+
+	// Create the forks
+	let fork_1 = propose_and_import_blocks(BlockId::Hash(canon_hashes[0]), 10);
+	let fork_2 = propose_and_import_blocks(BlockId::Hash(canon_hashes[12]), 15);
+	let fork_3 = propose_and_import_blocks(BlockId::Hash(canon_hashes[18]), 10);
+
+	// We should be tracking a total of 9 epochs in the fork tree
+	assert_eq!(
+		epoch_changes.lock().tree().iter().count(),
+		9,
+	);
+
+	// And only one root
+	assert_eq!(
+		epoch_changes.lock().tree().roots().count(),
+		1,
+	);
+
+	// We finalize block #13 from the canon chain, so on the next epoch
+	// change the tree should be pruned, to not contain F (#7).
+	client.finalize_block(BlockId::Hash(canon_hashes[12]), None, false).unwrap();
+	propose_and_import_blocks(BlockId::Hash(client.info().chain.best_hash), 7);
+
+	// at this point no hashes from the first fork must exist on the tree
+	assert!(
+		!epoch_changes.lock().tree().iter().map(|(h, _, _)| h).any(|h| fork_1.contains(h)),
+	);
+
+	// but the epoch changes from the other forks must still exist
+	assert!(
+		epoch_changes.lock().tree().iter().map(|(h, _, _)| h).any(|h| fork_2.contains(h))
+	);
+
+	assert!(
+		epoch_changes.lock().tree().iter().map(|(h, _, _)| h).any(|h| fork_3.contains(h)),
+	);
+
+	// finalizing block #25 from the canon chain should prune out the second fork
+	client.finalize_block(BlockId::Hash(canon_hashes[24]), None, false).unwrap();
+	propose_and_import_blocks(BlockId::Hash(client.info().chain.best_hash), 8);
+
+	// at this point no hashes from the second fork must exist on the tree
+	assert!(
+		!epoch_changes.lock().tree().iter().map(|(h, _, _)| h).any(|h| fork_2.contains(h)),
+	);
+
+	// while epoch changes from the last fork should still exist
+	assert!(
+		epoch_changes.lock().tree().iter().map(|(h, _, _)| h).any(|h| fork_3.contains(h)),
+	);
+}
+
+#[test]
+#[should_panic]
+fn verify_slots_are_strictly_increasing() {
+	let mut net = BabeTestNet::new(1);
+
+	let peer = net.peer(0);
+	let data = peer.data.as_ref().expect("babe link set up during initialization");
+
+	let client = peer.client().as_full().expect("Only full clients are used in tests").clone();
+	let mut block_import = data.block_import.lock().take().expect("import set up during init");
+
+	let mut proposer_factory = DummyFactory {
+		client: client.clone(),
+		config: data.link.config.clone(),
+		epoch_changes: data.link.epoch_changes.clone(),
+		mutator: Arc::new(|_, _| ()),
+	};
+
+	let genesis_header = client.header(&BlockId::Number(0)).unwrap().unwrap();
+
+	// we should have no issue importing this block
+	let b1 = propose_and_import_block(
+		&genesis_header,
+		Some(999),
+		&mut proposer_factory,
+		&mut block_import,
+	);
+
+	let b1 = client.header(&BlockId::Hash(b1)).unwrap().unwrap();
+
+	// we should fail to import this block since the slot number didn't increase.
+	// we will panic due to the `PanickingBlockImport` defined above.
+	propose_and_import_block(
+		&b1,
+		Some(999),
+		&mut proposer_factory,
+		&mut block_import,
+	);
+}

substrate/client/consensus/babe/src/verification.rs

@@ -0,0 +1,217 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Verification for BABE headers.
+use schnorrkel::vrf::{VRFOutput, VRFProof};
+use sr_primitives::{traits::Header, traits::DigestItemFor};
+use primitives::{Pair, Public};
+use babe_primitives::{Epoch, BabePreDigest, CompatibleDigestItem, AuthorityId};
+use babe_primitives::{AuthoritySignature, SlotNumber, AuthorityIndex, AuthorityPair};
+use slots::CheckedHeader;
+use log::{debug, trace};
+use super::{find_pre_digest, babe_err, BlockT, Error};
+use super::authorship::{make_transcript, calculate_primary_threshold, check_primary_threshold, secondary_slot_author};
+
+/// BABE verification parameters
+pub(super) struct VerificationParams<'a, B: 'a + BlockT> {
+	/// the header being verified.
+	pub(super) header: B::Header,
+	/// the pre-digest of the header being verified. this is optional - if prior
+	/// verification code had to read it, it can be included here to avoid duplicate
+	/// work.
+	pub(super) pre_digest: Option<BabePreDigest>,
+	/// the slot number of the current time.
+	pub(super) slot_now: SlotNumber,
+	/// epoch descriptor of the epoch this block _should_ be under, if it's valid.
+	pub(super) epoch: &'a Epoch,
+	/// genesis config of this BABE chain.
+	pub(super) config: &'a super::Config,
+}
+
+/// Check a header has been signed by the right key. If the slot is too far in
+/// the future, an error will be returned. If successful, returns the pre-header
+/// and the digest item containing the seal.
+///
+/// The seal must be the last digest.  Otherwise, the whole header is considered
+/// unsigned.  This is required for security and must not be changed.
+///
+/// This digest item will always return `Some` when used with `as_babe_pre_digest`.
+///
+/// The given header can either be from a primary or secondary slot assignment,
+/// with each having different validation logic.
+pub(super) fn check_header<B: BlockT + Sized>(
+	params: VerificationParams<B>,
+) -> Result<CheckedHeader<B::Header, VerifiedHeaderInfo<B>>, Error<B>> where
+	DigestItemFor<B>: CompatibleDigestItem,
+{
+	let VerificationParams {
+		mut header,
+		pre_digest,
+		slot_now,
+		epoch,
+		config,
+	} = params;
+
+	let authorities = &epoch.authorities;
+	let pre_digest = pre_digest.map(Ok).unwrap_or_else(|| find_pre_digest::<B>(&header))?;
+
+	trace!(target: "babe", "Checking header");
+	let seal = match header.digest_mut().pop() {
+		Some(x) => x,
+		None => return Err(babe_err(Error::HeaderUnsealed(header.hash()))),
+	};
+
+	let sig = seal.as_babe_seal().ok_or_else(|| {
+		babe_err(Error::HeaderBadSeal(header.hash()))
+	})?;
+
+	// the pre-hash of the header doesn't include the seal
+	// and that's what we sign
+	let pre_hash = header.hash();
+
+	if pre_digest.slot_number() > slot_now {
+		header.digest_mut().push(seal);
+		return Ok(CheckedHeader::Deferred(header, pre_digest.slot_number()));
+	}
+
+	let author = match authorities.get(pre_digest.authority_index() as usize) {
+		Some(author) => author.0.clone(),
+		None => return Err(babe_err(Error::SlotAuthorNotFound)),
+	};
+
+	match &pre_digest {
+		BabePreDigest::Primary { vrf_output, vrf_proof, authority_index, slot_number } => {
+			debug!(target: "babe", "Verifying Primary block");
+
+			let digest = (vrf_output, vrf_proof, *authority_index, *slot_number);
+
+			check_primary_header::<B>(
+				pre_hash,
+				digest,
+				sig,
+				&epoch,
+				config.c,
+			)?;
+		},
+		BabePreDigest::Secondary { authority_index, slot_number } if config.secondary_slots => {
+			debug!(target: "babe", "Verifying Secondary block");
+
+			let digest = (*authority_index, *slot_number);
+
+			check_secondary_header::<B>(
+				pre_hash,
+				digest,
+				sig,
+				&epoch,
+			)?;
+		},
+		_ => {
+			return Err(babe_err(Error::SecondarySlotAssignmentsDisabled));
+		}
+	}
+
+	let info = VerifiedHeaderInfo {
+		pre_digest: CompatibleDigestItem::babe_pre_digest(pre_digest),
+		seal,
+		author,
+	};
+	Ok(CheckedHeader::Checked(header, info))
+}
+
+pub(super) struct VerifiedHeaderInfo<B: BlockT> {
+	pub(super) pre_digest: DigestItemFor<B>,
+	pub(super) seal: DigestItemFor<B>,
+	pub(super) author: AuthorityId,
+}
+
+/// Check a primary slot proposal header. We validate that the given header is
+/// properly signed by the expected authority, and that the contained VRF proof
+/// is valid. Additionally, the weight of this block must increase compared to
+/// its parent since it is a primary block.
+fn check_primary_header<B: BlockT + Sized>(
+	pre_hash: B::Hash,
+	pre_digest: (&VRFOutput, &VRFProof, AuthorityIndex, SlotNumber),
+	signature: AuthoritySignature,
+	epoch: &Epoch,
+	c: (u64, u64),
+) -> Result<(), Error<B>> {
+	let (vrf_output, vrf_proof, authority_index, slot_number) = pre_digest;
+
+	let author = &epoch.authorities[authority_index as usize].0;
+
+	if AuthorityPair::verify(&signature, pre_hash, &author) {
+		let (inout, _) = {
+			let transcript = make_transcript(
+				&epoch.randomness,
+				slot_number,
+				epoch.epoch_index,
+			);
+
+			schnorrkel::PublicKey::from_bytes(author.as_slice()).and_then(|p| {
+				p.vrf_verify(transcript, vrf_output, vrf_proof)
+			}).map_err(|s| {
+				babe_err(Error::VRFVerificationFailed(s))
+			})?
+		};
+
+		let threshold = calculate_primary_threshold(
+			c,
+			&epoch.authorities,
+			authority_index as usize,
+		);
+
+		if !check_primary_threshold(&inout, threshold) {
+			return Err(babe_err(Error::VRFVerificationOfBlockFailed(author.clone(), threshold)));
+		}
+
+		Ok(())
+	} else {
+		Err(babe_err(Error::BadSignature(pre_hash)))
+	}
+}
+
+/// Check a secondary slot proposal header. We validate that the given header is
+/// properly signed by the expected authority, which we have a deterministic way
+/// of computing. Additionally, the weight of this block must stay the same
+/// compared to its parent since it is a secondary block.
+fn check_secondary_header<B: BlockT>(
+	pre_hash: B::Hash,
+	pre_digest: (AuthorityIndex, SlotNumber),
+	signature: AuthoritySignature,
+	epoch: &Epoch,
+) -> Result<(), Error<B>> {
+	let (authority_index, slot_number) = pre_digest;
+
+	// check the signature is valid under the expected authority and
+	// chain state.
+	let expected_author = secondary_slot_author(
+		slot_number,
+		&epoch.authorities,
+		epoch.randomness,
+	).ok_or_else(|| Error::NoSecondaryAuthorExpected)?;
+
+	let author = &epoch.authorities[authority_index as usize].0;
+
+	if expected_author != author {
+		return Err(Error::InvalidAuthor(expected_author.clone(), author.clone()));
+	}
+
+	if AuthorityPair::verify(&signature, pre_hash.as_ref(), author) {
+		Ok(())
+	} else {
+		Err(Error::BadSignature(pre_hash))
+	}
+}