Allow for customisation of chain selection systems (#2240)

* move SelectChain trait out of client

* Extend SelectChain, move longest chain implementation into it

* Bring SelectChain into service

* implement LongestChain SelectChain

* implement longest chain for node

* update Cargo.lock's

* in between erroring tests

* deprecate ::backend and ::import_lock

* Remove unneded space

Co-Authored-By: gnunicorn <ben.kampmann@googlemail.com>

* Remove unneded space

Co-Authored-By: gnunicorn <ben.kampmann@googlemail.com>

* Fixes test compilation

* remove todo

* re-enable client test

* add doc

* fixing tests

* Clarify SelectChain Interface, intended implementation and usage

* minor components cleanups

* minor cleanups

* Update lock files

* Implement cleaner interface for SelectChain

* addressing comments

* Updating tests

* bump node runtime impl version

* address grumbles

substrate/core/consensus/aura/src/lib.rs

@@ -25,18 +25,16 @@
 //!
 //! Blocks from future steps will be either deferred or rejected depending on how
 //! far in the future they are.
-#![deny(warnings)]
 #![forbid(missing_docs, unsafe_code)]
 use std::{sync::Arc, time::Duration, thread, marker::PhantomData, hash::Hash, fmt::Debug};
 
 use parity_codec::{Encode, Decode};
 use consensus_common::{self, Authorities, BlockImport, Environment, Proposer,
 	ForkChoiceStrategy, ImportBlock, BlockOrigin, Error as ConsensusError,
+	SelectChain, well_known_cache_keys
 };
-use consensus_common::well_known_cache_keys;
 use consensus_common::import_queue::{Verifier, BasicQueue, SharedBlockImport, SharedJustificationImport};
 use client::{
-	ChainHead,
 	block_builder::api::BlockBuilder as BlockBuilderApi,
 	blockchain::ProvideCache,
 	runtime_api::{ApiExt, Core as CoreApi},
@@ -182,10 +180,11 @@ impl SlotCompatible for AuraSlotCompatible {
 
 /// Start the aura worker in a separate thread.
 #[deprecated(since = "1.1", note = "Please spawn a thread manually")]
-pub fn start_aura_thread<B, C, E, I, P, SO, Error, OnExit>(
+pub fn start_aura_thread<B, C, SC, E, I, P, SO, Error, OnExit>(
 	slot_duration: SlotDuration,
 	local_key: Arc<P>,
 	client: Arc<C>,
+	select_chain: SC,
 	block_import: Arc<I>,
 	env: Arc<E>,
 	sync_oracle: SO,
@@ -194,8 +193,9 @@ pub fn start_aura_thread<B, C, E, I, P, SO, Error, OnExit>(
 	force_authoring: bool,
 ) -> Result<(), consensus_common::Error> where
 	B: Block + 'static,
-	C: ChainHead<B> + ProvideRuntimeApi + ProvideCache<B> + Send + Sync + 'static,
+	C: ProvideRuntimeApi + ProvideCache<B> + Send + Sync + 'static,
 	C::Api: AuthoritiesApi<B>,
+	SC: SelectChain<B> + Clone + 'static,
 	E: Environment<B, Error=Error> + Send + Sync + 'static,
 	E::Proposer: Proposer<B, Error=Error> + Send + 'static,
 	<<E::Proposer as Proposer<B>>::Create as IntoFuture>::Future: Send + 'static,
@@ -222,7 +222,7 @@ pub fn start_aura_thread<B, C, E, I, P, SO, Error, OnExit>(
 	#[allow(deprecated)]	// The function we are in is also deprecated.
 	slots::start_slot_worker_thread::<_, _, _, _, AuraSlotCompatible, u64, _>(
 		slot_duration.0,
-		client,
+		select_chain,
 		Arc::new(worker),
 		sync_oracle,
 		on_exit,
@@ -231,10 +231,11 @@ pub fn start_aura_thread<B, C, E, I, P, SO, Error, OnExit>(
 }
 
 /// Start the aura worker. The returned future should be run in a tokio runtime.
-pub fn start_aura<B, C, E, I, P, SO, Error, OnExit>(
+pub fn start_aura<B, C, SC, E, I, P, SO, Error, OnExit>(
 	slot_duration: SlotDuration,
 	local_key: Arc<P>,
 	client: Arc<C>,
+	select_chain: SC,
 	block_import: Arc<I>,
 	env: Arc<E>,
 	sync_oracle: SO,
@@ -243,8 +244,9 @@ pub fn start_aura<B, C, E, I, P, SO, Error, OnExit>(
 	force_authoring: bool,
 ) -> Result<impl Future<Item=(), Error=()>, consensus_common::Error> where
 	B: Block,
-	C: ChainHead<B> + ProvideRuntimeApi + ProvideCache<B>,
+	C: ProvideRuntimeApi + ProvideCache<B>,
 	C::Api: AuthoritiesApi<B>,
+	SC: SelectChain<B> + Clone,
 	E: Environment<B, Error=Error>,
 	E::Proposer: Proposer<B, Error=Error>,
 	<<E::Proposer as Proposer<B>>::Create as IntoFuture>::Future: Send + 'static,
@@ -268,7 +270,7 @@ pub fn start_aura<B, C, E, I, P, SO, Error, OnExit>(
 	};
 	slots::start_slot_worker::<_, _, _, _, _, AuraSlotCompatible, _>(
 		slot_duration.0,
-		client,
+		select_chain,
 		Arc::new(worker),
 		sync_oracle,
 		on_exit,
@@ -804,7 +806,7 @@ mod tests {
 	use tokio::runtime::current_thread;
 	use keyring::sr25519::Keyring;
 	use primitives::sr25519;
-	use client::BlockchainEvents;
+	use client::{LongestChain, BlockchainEvents};
 	use test_client;
 
 	type Error = client::error::Error;
@@ -916,6 +918,10 @@ mod tests {
 		let mut runtime = current_thread::Runtime::new().unwrap();
 		for (peer_id, key) in peers {
 			let client = net.lock().peer(*peer_id).client().clone();
+			let select_chain = LongestChain::new(
+				client.backend().clone(),
+				client.import_lock().clone(),
+			);
 			let environ = Arc::new(DummyFactory(client.clone()));
 			import_notifications.push(
 				client.import_notification_stream()
@@ -931,10 +937,11 @@ mod tests {
 				&inherent_data_providers, slot_duration.get()
 			).expect("Registers aura inherent data provider");
 
-			let aura = start_aura::<_, _, _, _, sr25519::Pair, _, _, _>(
+			let aura = start_aura::<_, _, _, _, _, sr25519::Pair, _, _, _>(
 				slot_duration,
 				Arc::new(key.clone().into()),
 				client.clone(),
+				select_chain,
 				client,
 				environ.clone(),
 				DummyOracle,

substrate/core/consensus/babe/src/lib.rs

@@ -58,10 +58,9 @@ use srml_babe::{
 	BabeInherentData,
 	timestamp::{TimestampInherentData, InherentType as TimestampInherent}
 };
-use consensus_common::well_known_cache_keys;
+use consensus_common::{SelectChain, well_known_cache_keys};
 use consensus_common::import_queue::{Verifier, BasicQueue};
 use client::{
-	ChainHead,
 	block_builder::api::BlockBuilder as BlockBuilderApi,
 	blockchain::ProvideCache,
 	runtime_api::ApiExt,
@@ -249,7 +248,7 @@ impl SlotCompatible for BabeSlotCompatible {
 }
 
 /// Parameters for BABE.
-pub struct BabeParams<C, E, I, SO, OnExit> {
+pub struct BabeParams<C, E, I, SO, SC, OnExit> {
 
 	/// The configuration for BABE.  Includes the slot duration, threshold, and
 	/// other parameters.
@@ -261,6 +260,9 @@ pub struct BabeParams<C, E, I, SO, OnExit> {
 	/// The client to use
 	pub client: Arc<C>,
 
+	/// The SelectChain Strategy
+	pub select_chain: SC,
+
 	/// A block importer
 	pub block_import: Arc<I>,
 
@@ -281,28 +283,30 @@ pub struct BabeParams<C, E, I, SO, OnExit> {
 }
 
 /// Start the babe worker. The returned future should be run in a tokio runtime.
-pub fn start_babe<B, C, E, I, SO, Error, OnExit>(BabeParams {
+pub fn start_babe<B, C, E, I, SO, SC, Error, OnExit>(BabeParams {
 	config,
 	local_key,
 	client,
+	select_chain,
 	block_import,
 	env,
 	sync_oracle,
 	on_exit,
 	inherent_data_providers,
 	force_authoring,
-}: BabeParams<C, E, I, SO, OnExit>) -> Result<
+}: BabeParams<C, E, I, SO, SC, OnExit>) -> Result<
 	impl Future<Item=(), Error=()>,
 	consensus_common::Error,
 > where
 	B: Block,
-	C: ChainHead<B> + ProvideRuntimeApi + ProvideCache<B>,
+	C: ProvideRuntimeApi + ProvideCache<B>,
 	C::Api: AuthoritiesApi<B>,
 	E: Environment<B, Error=Error>,
 	E::Proposer: Proposer<B, Error=Error>,
 	<<E::Proposer as Proposer<B>>::Create as IntoFuture>::Future: Send + 'static,
 	I: BlockImport<B> + Send + Sync + 'static,
 	SO: SyncOracle + Send + Sync + Clone,
+	SC: SelectChain<B>,
 	DigestItemFor<B>: CompatibleDigestItem + DigestItem<AuthorityId=Public>,
 	Error: ::std::error::Error + Send + From<::consensus_common::Error> + From<I::Error> + 'static,
 	OnExit: Future<Item=(), Error=()>,
@@ -319,7 +323,7 @@ pub fn start_babe<B, C, E, I, SO, Error, OnExit>(BabeParams {
 	};
 	slots::start_slot_worker::<_, _, _, _, _, BabeSlotCompatible, _>(
 		config.0,
-		client,
+		select_chain,
 		Arc::new(worker),
 		sync_oracle,
 		on_exit,
@@ -847,10 +851,13 @@ fn claim_slot(
 }
 
 #[cfg(test)]
-#[allow(dead_code, unused_imports)]
+#[allow(dead_code, unused_imports, deprecated)]
+// FIXME #2532: need to allow deprecated until refactor is done https://github.com/paritytech/substrate/issues/2532
+
 mod tests {
 	use super::*;
 
+	use client::LongestChain;
 	use consensus_common::NoNetwork as DummyOracle;
 	use network::test::*;
 	use network::test::{Block as TestBlock, PeersClient};
@@ -1014,6 +1021,7 @@ mod tests {
 				config,
 				local_key: Arc::new(key.clone().into()),
 				block_import: client.clone(),
+				select_chain: LongestChain::new(client.backend().clone(), client.import_lock().clone()),
 				client,
 				env: environ.clone(),
 				sync_oracle: DummyOracle,

substrate/core/consensus/common/src/error.rs

@@ -105,5 +105,11 @@ error_chain! {
 			description("Import failed"),
 			display("Import failed: {}", reason),
 		}
+
+		/// Error from the client while importing
+		ChainLookup(reason: String) {
+			description("Looking up chain failed"),
+			display("Chain lookup failed: {}", reason),
+		}
 	}
 }

substrate/core/consensus/common/src/lib.rs

@@ -40,6 +40,7 @@ pub use inherents::InherentData;
 pub mod offline_tracker;
 pub mod error;
 mod block_import;
+mod select_chain;
 pub mod import_queue;
 pub mod evaluation;
 
@@ -50,6 +51,7 @@ pub use self::error::{Error, ErrorKind};
 pub use block_import::{
 	BlockImport, BlockOrigin, ForkChoiceStrategy, ImportedAux, ImportBlock, ImportResult, JustificationImport,
 };
+pub use select_chain::SelectChain;
 
 /// Trait for getting the authorities at a given block.
 pub trait Authorities<B: Block> {

substrate/core/consensus/common/src/select_chain.rs

@@ -0,0 +1,54 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate Consensus Common.
+
+// Substrate Demo is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate Consensus Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate Consensus Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::error::Error;
+use runtime_primitives::traits::{Block as BlockT, NumberFor};
+
+
+/// The SelectChain trait defines the strategy upon which the head is chosen
+/// if multiple forks are present for an opaque definition of "best" in the 
+/// specific chain build.
+///
+/// The Strategy can be customised for the two use cases of authoring new blocks
+/// upon the best chain or which fork to finalise. Unless implemented differently
+/// by default finalisation methods fall back to use authoring, so as a minimum
+/// `_authoring`-functions must be implemented. 
+///
+/// Any particular user must make explicit, however, whether they intend to finalise
+/// or author through the using the right function call, as these might differ in
+/// some implementations.
+///
+/// Non-deterministicly finalising chains may only use the `_authoring` functions.
+pub trait SelectChain<Block: BlockT>: Sync + Send + Clone {
+
+	/// Get all leaves of the chain: block hashes that have no children currently.
+	/// Leaves that can never be finalized will not be returned.
+	fn leaves(&self) -> Result<Vec<<Block as BlockT>::Hash>, Error>;
+
+	/// Among those `leaves` deterministically pick one chain as the generally
+	/// best chain to author new blocks upon and probably finalize.
+	fn best_chain(&self) -> Result<<Block as BlockT>::Header, Error>;
+
+	/// Get the best ancestor of `target_hash` that we should attempt
+	/// to finalize next.
+	fn finality_target(
+		&self,
+		target_hash: <Block as BlockT>::Hash,
+		_maybe_max_number: Option<NumberFor<Block>>
+	) -> Result<Option<<Block as BlockT>::Hash>, Error> {
+		Ok(Some(target_hash))
+	}
+}

substrate/core/consensus/rhd/src/service.rs

@@ -22,7 +22,7 @@ use std::thread;
 use std::time::{Duration, Instant};
 use std::sync::Arc;
 
-use client::{BlockchainEvents, ChainHead, BlockBody};
+use client::{BlockchainEvents, BlockBody};
 use futures::prelude::*;
 use transaction_pool::txpool::{Pool as TransactionPool, ChainApi as PoolChainApi};
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, BlockNumberToHash};
@@ -33,7 +33,7 @@ use tokio::runtime::current_thread::Runtime as LocalRuntime;
 use tokio::timer::Interval;
 
 use parking_lot::RwLock;
-use consensus::offline_tracker::OfflineTracker;
+use consensus::{self, offline_tracker::OfflineTracker};
 
 use super::{Network, ProposerFactory, AuthoringApi};
 use {consensus, primitives, ed25519, error, BftService, LocalProposer};
@@ -87,9 +87,9 @@ impl Service {
 			A: AuthoringApi + BlockNumberToHash + 'static,
 			P: PoolChainApi<Block = <A as AuthoringApi>::Block> + 'static,
 			C: BlockchainEvents<<A as AuthoringApi>::Block>
-				+ ChainHead<<A as AuthoringApi>::Block>
-				+ BlockBody<<A as AuthoringApi>::Block>,
-			C: consensus::BlockImport<<A as AuthoringApi>::Block>
+				+ BlockBody<<A as AuthoringApi>::Block>
+				+ consensus::SelectChain<<A as AuthoringApi>::Block>
+				+ consensus::BlockImport<<A as AuthoringApi>::Block>
 				+ consensus::Authorities<<A as AuthoringApi>::Block> + Send + Sync + 'static,
 			primitives::H256: From<<<A as AuthoringApi>::Block as BlockT>::Hash>,
 			<<A as AuthoringApi>::Block as BlockT>::Hash: PartialEq<primitives::H256> + PartialEq,

substrate/core/consensus/slots/src/lib.rs

@@ -26,9 +26,8 @@ mod slots;
 
 pub use slots::{slot_now, SlotInfo, Slots};
 
-use client::ChainHead;
 use codec::{Decode, Encode};
-use consensus_common::SyncOracle;
+use consensus_common::{SyncOracle, SelectChain};
 use futures::prelude::*;
 use futures::{
 	future::{self, Either},
@@ -73,7 +72,7 @@ pub fn inherent_to_common_error(err: inherents::RuntimeString) -> consensus_comm
 #[deprecated(since = "1.1", note = "Please spawn a thread manually")]
 pub fn start_slot_worker_thread<B, C, W, SO, SC, T, OnExit>(
 	slot_duration: SlotDuration<T>,
-	client: Arc<C>,
+	select_chain: C,
 	worker: Arc<W>,
 	sync_oracle: SO,
 	on_exit: OnExit,
@@ -81,7 +80,7 @@ pub fn start_slot_worker_thread<B, C, W, SO, SC, T, OnExit>(
 ) -> Result<(), consensus_common::Error>
 where
 	B: Block + 'static,
-	C: ChainHead<B> + Send + Sync + 'static,
+	C: SelectChain<B> + Clone + 'static,
 	W: SlotWorker<B> + Send + Sync + 'static,
 	SO: SyncOracle + Send + Clone + 'static,
 	SC: SlotCompatible + 'static,
@@ -101,9 +100,9 @@ where
 			}
 		};
 
-		let slot_worker_future = match start_slot_worker::<_, _, _, _, _, SC, _>(
+		let slot_worker_future = match start_slot_worker::<_, _, _, T, _, SC, _>(
 			slot_duration.clone(),
-			client,
+			select_chain,
 			worker,
 			sync_oracle,
 			on_exit,
@@ -134,7 +133,7 @@ where
 /// Start a new slot worker.
 pub fn start_slot_worker<B, C, W, T, SO, SC, OnExit>(
 	slot_duration: SlotDuration<T>,
-	client: Arc<C>,
+	client: C,
 	worker: Arc<W>,
 	sync_oracle: SO,
 	on_exit: OnExit,
@@ -142,7 +141,7 @@ pub fn start_slot_worker<B, C, W, T, SO, SC, OnExit>(
 ) -> Result<impl Future<Item = (), Error = ()>, consensus_common::Error>
 where
 	B: Block,
-	C: ChainHead<B>,
+	C: SelectChain<B> + Clone,
 	W: SlotWorker<B>,
 	SO: SyncOracle + Send + Clone,
 	SC: SlotCompatible,
@@ -173,7 +172,7 @@ where
 				}
 
 				let slot_num = slot_info.number;
-				let chain_head = match client.best_block_header() {
+				let chain_head = match client.best_chain() {
 					Ok(x) => x,
 					Err(e) => {
 						warn!(target: "slots", "Unable to author block in slot {}. \