New sessions, kill consensus module (#2802)

* Draft of new sessions

* Reintroduce tuple impls

* Move staking module to new session API

* More work on staking and grandpa.

* Use iterator to avoid cloning and tuple macro

* Make runtime build again

* Polish the OpaqueKeys devex

* Move consensus logic into system & aura.

* Fix up system module

* Get build mostly going. Stuck at service.rs

* Building again

* Update srml/staking/src/lib.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Refactoring out Consensus module, AuthorityIdOf, &c.

* Refactored out DigestItem::AuthoritiesChanged. Building.

* Remove tentative code

* Remove invalid comment

* Make Seal opaque and introduce nice methods for handling opaque items.

* Start to use proper digest for Aura authorities tracking.

* Fix up grandpa, remove system::Raw/Log

* Refactor Grandpa to use new logging infrastructure.

Also make authorityid/sessionkey static. Switch over to storing
authorities in a straight Vec.

* Building again

* Tidy up some AuthorityIds

* Expunge most of the rest of the AuthorityKey confusion.

Also, de-generify Babe and re-generify Aura.

* Remove cruft

* Untangle last of the `AuthorityId`s.

* Sort out finality_tracker

* Refactor median getting

* Apply suggestions from code review

Co-Authored-By: Robert Habermeier <rphmeier@gmail.com>

* Session tests works

* Update core/sr-primitives/src/generic/digest.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Session tests works

* Fix for staking from @dvc94ch

* log an error

* fix test runtime build

* Some test fixes

* Staking mock update to new session api.

* Fix build.

* Move OpaqueKeys to primitives.

* Use on_initialize instead of check_rotate_session.

* Update tests to new staking api.

* fixup mock

* Fix bond_extra_and_withdraw_unbonded_works.

* Fix bond_with_little_staked_value_bounded_by_slot_stake.

* Fix bond_with_no_staked_value.

* Fix change_controller_works.

* Fix less_than_needed_candidates_works.

* Fix multi_era_reward_should_work.

* Fix nominating_and_rewards_should_work.

* Fix nominators_also_get_slashed.

* Fix phragmen_large_scale_test.

* Fix phragmen_poc_works.

* Fix phragmen_score_should_be_accurate_on_large_stakes.

* Fix phragmen_should_not_overflow.

* Fix reward_destination_works.

* Fix rewards_should_work.

* Fix sessions_and_eras_should_work.

* Fix slot_stake_is_least_staked_validator.

* Fix too_many_unbond_calls_should_not_work.

* Fix wrong_vote_is_null.

* Fix runtime.

* Fix wasm runtime build.

* Update Cargo.lock

* Fix warnings.

* Fix grandpa tests.

* Fix test-runtime build.

* Fix template node build.

* Fix stuff.

* Update Cargo.lock to fix CI

* Re-add missing AuRa logs

Runtimes are required to know about every digest they receive ― they
panic otherwise.  This re-adds support for AuRa pre-runtime digests.

* Update core/consensus/babe/src/digest.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Kill log trait and all that jazz.

* Refactor staking tests.

* Fix ci runtime wasm check.

* Line length 120.

* Make tests build again

* Remove trailing commas in function declarations

The `extern_functions!` macro doesn’t like them, perhaps due to a bug in
rustc.

* Fix type error

* Fix compilation errors

* Fix a test

* Another couple of fixes

* Fix another test

* More test fixes

* Another test fix

* Bump runtime.

* Wrap long line

* Fix build, remove redundant code.

* Issue to track TODO

* Leave the benchmark code alone.

* Fix missing `std::time::{Instant, Duration}`

* Indentation

* Aura ConsensusLog as enum

substrate/core/consensus/aura/src/digest.rs

@@ -18,20 +18,22 @@
 //!
 //! This implements the digests for AuRa, to allow the private
 //! `CompatibleDigestItem` trait to appear in public interfaces.
+
 use primitives::Pair;
 use aura_primitives::AURA_ENGINE_ID;
-use runtime_primitives::generic::DigestItem;
-use parity_codec::{Encode, Decode};
+use runtime_primitives::generic::{DigestItem, OpaqueDigestItemId};
+use parity_codec::{Encode, Codec};
+use std::fmt::Debug;
 
 type Signature<P> = <P as Pair>::Signature;
 
 /// A digest item which is usable with aura consensus.
-pub trait CompatibleDigestItem<T: Pair>: Sized {
+pub trait CompatibleDigestItem<P: Pair>: Sized {
 	/// Construct a digest item which contains a signature on the hash.
-	fn aura_seal(signature: Signature<T>) -> Self;
+	fn aura_seal(signature: Signature<P>) -> Self;
 
 	/// If this item is an Aura seal, return the signature.
-	fn as_aura_seal(&self) -> Option<&Signature<T>>;
+	fn as_aura_seal(&self) -> Option<Signature<P>>;
 
 	/// Construct a digest item which contains the slot number
 	fn aura_pre_digest(slot_num: u64) -> Self;
@@ -40,18 +42,17 @@ pub trait CompatibleDigestItem<T: Pair>: Sized {
 	fn as_aura_pre_digest(&self) -> Option<u64>;
 }
 
-impl<P, Hash> CompatibleDigestItem<P> for DigestItem<Hash, P::Public, P::Signature>
-	where P: Pair, P::Signature: Clone + Encode + Decode,
+impl<P, Hash> CompatibleDigestItem<P> for DigestItem<Hash> where
+	P: Pair,
+	Signature<P>: Codec,
+	Hash: Debug + Send + Sync + Eq + Clone + Codec + 'static
 {
 	fn aura_seal(signature: Signature<P>) -> Self {
-		DigestItem::Seal(AURA_ENGINE_ID, signature)
+		DigestItem::Seal(AURA_ENGINE_ID, signature.encode())
 	}
 
-	fn as_aura_seal(&self) -> Option<&Signature<P>> {
-		match self {
-			DigestItem::Seal(AURA_ENGINE_ID, ref sig) => Some(sig),
-			_ => None,
-		}
+	fn as_aura_seal(&self) -> Option<Signature<P>> {
+		self.try_to(OpaqueDigestItemId::Seal(&AURA_ENGINE_ID))
 	}
 
 	fn aura_pre_digest(slot_num: u64) -> Self {
@@ -59,9 +60,6 @@ impl<P, Hash> CompatibleDigestItem<P> for DigestItem<Hash, P::Public, P::Signatu
 	}
 
 	fn as_aura_pre_digest(&self) -> Option<u64> {
-		match self {
-			DigestItem::PreRuntime(AURA_ENGINE_ID, ref buffer) => Decode::decode(&mut &buffer[..]),
-			_ => None,
-		}
+		self.try_to(OpaqueDigestItemId::PreRuntime(&AURA_ENGINE_ID))
 	}
 }

substrate/core/consensus/aura/src/lib.rs

@@ -25,13 +25,15 @@
 //!
 //! Blocks from future steps will be either deferred or rejected depending on how
 //! far in the future they are.
+//!
+//! NOTE: Aura itself is designed to be generic over the crypto used.
 #![forbid(missing_docs, unsafe_code)]
 use std::{sync::Arc, time::Duration, thread, marker::PhantomData, hash::Hash, fmt::Debug};
 
-use parity_codec::{Encode, Decode};
-use consensus_common::{self, Authorities, BlockImport, Environment, Proposer,
+use parity_codec::{Encode, Decode, Codec};
+use consensus_common::{self, BlockImport, Environment, Proposer,
 	ForkChoiceStrategy, ImportBlock, BlockOrigin, Error as ConsensusError,
-	SelectChain, well_known_cache_keys
+	SelectChain, well_known_cache_keys::{self, Id as CacheKeyId}
 };
 use consensus_common::import_queue::{
 	Verifier, BasicQueue, SharedBlockImport, SharedJustificationImport, SharedFinalityProofImport,
@@ -45,15 +47,11 @@ use client::{
 	backend::AuxStore,
 };
 
-use runtime_primitives::{generic::{self, BlockId}, Justification};
-use runtime_primitives::traits::{
-	Block, Header, Digest, DigestItemFor, DigestItem, ProvideRuntimeApi, AuthorityIdFor,
-	Zero, Member,
-};
+use runtime_primitives::{generic::{self, BlockId, OpaqueDigestItemId}, Justification};
+use runtime_primitives::traits::{Block, Header, DigestItemFor, ProvideRuntimeApi, Zero, Member};
 
 use primitives::Pair;
 use inherents::{InherentDataProviders, InherentData};
-use authorities::AuthoritiesApi;
 
 use futures::{Future, IntoFuture, future};
 use tokio_timer::Timeout;
@@ -82,9 +80,12 @@ pub struct SlotDuration(slots::SlotDuration<u64>);
 impl SlotDuration {
 	/// Either fetch the slot duration from disk or compute it from the genesis
 	/// state.
-	pub fn get_or_compute<B: Block, C>(client: &C) -> CResult<Self>
+	pub fn get_or_compute<A, B, C>(client: &C) -> CResult<Self>
 	where
-		C: AuxStore, C: ProvideRuntimeApi, C::Api: AuraApi<B>,
+		A: Codec,
+		B: Block,
+		C: AuxStore + ProvideRuntimeApi,
+		C::Api: AuraApi<B, A>,
 	{
 		slots::SlotDuration::get_or_compute(client, |a, b| a.slot_duration(b)).map(Self)
 	}
@@ -138,19 +139,14 @@ pub fn start_aura<B, C, SC, E, I, P, SO, Error, H>(
 ) -> Result<impl Future<Item=(), Error=()>, consensus_common::Error> where
 	B: Block<Header=H>,
 	C: ProvideRuntimeApi + ProvideCache<B> + AuxStore + Send + Sync,
-	C::Api: AuthoritiesApi<B>,
+	C::Api: AuraApi<B, AuthorityId<P>>,
 	SC: SelectChain<B>,
-	generic::DigestItem<B::Hash, P::Public, P::Signature>: DigestItem<Hash=B::Hash>,
 	E::Proposer: Proposer<B, Error=Error>,
 	<<E::Proposer as Proposer<B>>::Create as IntoFuture>::Future: Send + 'static,
 	P: Pair + Send + Sync + 'static,
 	P::Public: Hash + Member + Encode + Decode,
 	P::Signature: Hash + Member + Encode + Decode,
-	DigestItemFor<B>: CompatibleDigestItem<P> + DigestItem<AuthorityId=AuthorityId<P>>,
-	H: Header<
-		Digest=generic::Digest<generic::DigestItem<B::Hash, P::Public, P::Signature>>,
-		Hash=B::Hash,
-	>,
+	H: Header<Hash=B::Hash>,
 	E: Environment<B, Error=Error>,
 	I: BlockImport<B> + Send + Sync + 'static,
 	Error: ::std::error::Error + Send + From<::consensus_common::Error> + From<I::Error> + 'static,
@@ -189,20 +185,16 @@ struct AuraWorker<C, E, I, P, SO> {
 impl<H, B, C, E, I, P, Error, SO> SlotWorker<B> for AuraWorker<C, E, I, P, SO> where
 	B: Block<Header=H>,
 	C: ProvideRuntimeApi + ProvideCache<B> + Sync,
-	C::Api: AuthoritiesApi<B>,
+	C::Api: AuraApi<B, AuthorityId<P>>,
 	E: Environment<B, Error=Error>,
 	E::Proposer: Proposer<B, Error=Error>,
 	<<E::Proposer as Proposer<B>>::Create as IntoFuture>::Future: Send + 'static,
-	H: Header<
-		Digest=generic::Digest<generic::DigestItem<B::Hash, P::Public, P::Signature>>,
-		Hash=B::Hash,
-	>,
+	H: Header<Hash=B::Hash>,
 	I: BlockImport<B> + Send + Sync + 'static,
 	P: Pair + Send + Sync + 'static,
 	P::Public: Member + Encode + Decode + Hash,
 	P::Signature: Member + Encode + Decode + Hash + Debug,
 	SO: SyncOracle + Send + Clone,
-	DigestItemFor<B>: CompatibleDigestItem<P> + DigestItem<AuthorityId=AuthorityId<P>, Hash=B::Hash>,
 	Error: ::std::error::Error + Send + From<::consensus_common::Error> + From<I::Error> + 'static,
 {
 	type OnSlot = Box<dyn Future<Item=(), Error=consensus_common::Error> + Send>;
@@ -257,7 +249,7 @@ impl<H, B, C, E, I, P, Error, SO> SlotWorker<B> for AuraWorker<C, E, I, P, SO> w
 				);
 
 				// we are the slot author. make a block and sign it.
-				let proposer = match env.init(&chain_head, &authorities) {
+				let proposer = match env.init(&chain_head) {
 					Ok(p) => p,
 					Err(e) => {
 						warn!("Unable to author block in slot {:?}: {:?}", slot_num, e);
@@ -506,13 +498,12 @@ impl<C, P> AuraVerifier<C, P>
 
 #[forbid(deprecated)]
 impl<B: Block, C, P> Verifier<B> for AuraVerifier<C, P> where
-	C: ProvideRuntimeApi + Send + Sync + client::backend::AuxStore,
-	C::Api: BlockBuilderApi<B>,
-	DigestItemFor<B>: CompatibleDigestItem<P> + DigestItem<AuthorityId=AuthorityId<P>>,
+	C: ProvideRuntimeApi + Send + Sync + client::backend::AuxStore + ProvideCache<B>,
+	C::Api: BlockBuilderApi<B> + AuraApi<B, AuthorityId<P>>,
+	DigestItemFor<B>: CompatibleDigestItem<P>,
 	P: Pair + Send + Sync + 'static,
 	P::Public: Send + Sync + Hash + Eq + Clone + Decode + Encode + Debug + AsRef<P::Public> + 'static,
 	P::Signature: Encode + Decode,
-	Self: Authorities<B>,
 {
 	fn verify(
 		&self,
@@ -520,13 +511,13 @@ impl<B: Block, C, P> Verifier<B> for AuraVerifier<C, P> where
 		header: B::Header,
 		justification: Option<Justification>,
 		mut body: Option<Vec<B::Extrinsic>>,
-	) -> Result<(ImportBlock<B>, Option<Vec<AuthorityId<P>>>), String> {
+	) -> Result<(ImportBlock<B>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
 		let mut inherent_data = self.inherent_data_providers.create_inherent_data().map_err(String::from)?;
 		let (timestamp_now, slot_now) = AuraSlotCompatible::extract_timestamp_and_slot(&inherent_data)
 			.map_err(|e| format!("Could not extract timestamp and slot: {:?}", e))?;
 		let hash = header.hash();
 		let parent_hash = *header.parent_hash();
-		let authorities = self.authorities(&BlockId::Hash(parent_hash))
+		let authorities = authorities(self.client.as_ref(), &BlockId::Hash(parent_hash))
 			.map_err(|e| format!("Could not fetch authorities at {:?}: {:?}", parent_hash, e))?;
 
 		// we add one to allow for some small drift.
@@ -569,9 +560,14 @@ impl<B: Block, C, P> Verifier<B> for AuraVerifier<C, P> where
 				trace!(target: "aura", "Checked {:?}; importing.", pre_header);
 				telemetry!(CONSENSUS_TRACE; "aura.checked_and_importing"; "pre_header" => ?pre_header);
 
-				let new_authorities = pre_header.digest()
-					.log(DigestItem::as_authorities_change)
-					.map(|digest| digest.to_vec());
+				// `Consensus` is the Aura-specific authorities change log.
+				let maybe_keys = pre_header.digest()
+					.convert_first(|l| l.try_to::<ConsensusLog<AuthorityId<P>>>(
+						OpaqueDigestItemId::Consensus(&AURA_ENGINE_ID)
+					))
+					.map(|ConsensusLog::AuthoritiesChange(a)|
+						vec![(well_known_cache_keys::AUTHORITIES, a.encode())]
+					);
 
 				let import_block = ImportBlock {
 					origin,
@@ -584,7 +580,7 @@ impl<B: Block, C, P> Verifier<B> for AuraVerifier<C, P> where
 					fork_choice: ForkChoiceStrategy::LongestChain,
 				};
 
-				Ok((import_block, new_authorities))
+				Ok((import_block, maybe_keys))
 			}
 			CheckedHeader::Deferred(a, b) => {
 				debug!(target: "aura", "Checking {:?} failed; {:?}, {:?}.", hash, a, b);
@@ -597,22 +593,11 @@ impl<B: Block, C, P> Verifier<B> for AuraVerifier<C, P> where
 	}
 }
 
-impl<B, C, P> Authorities<B> for AuraVerifier<C, P> where
+fn initialize_authorities_cache<A, B, C>(client: &C) -> Result<(), ConsensusError> where
+	A: Codec,
 	B: Block,
 	C: ProvideRuntimeApi + ProvideCache<B>,
-	C::Api: AuthoritiesApi<B>,
-{
-	type Error = ConsensusError;
-
-	fn authorities(&self, at: &BlockId<B>) -> Result<Vec<AuthorityIdFor<B>>, Self::Error> {
-		authorities(self.client.as_ref(), at)
-	}
-}
-
-fn initialize_authorities_cache<B, C>(client: &C) -> Result<(), ConsensusError> where
-	B: Block,
-	C: ProvideRuntimeApi + ProvideCache<B>,
-	C::Api: AuthoritiesApi<B>,
+	C::Api: AuraApi<B, A>,
 {
 	// no cache => no initialization
 	let cache = match client.cache() {
@@ -622,7 +607,7 @@ fn initialize_authorities_cache<B, C>(client: &C) -> Result<(), ConsensusError>
 
 	// check if we already have initialized the cache
 	let genesis_id = BlockId::Number(Zero::zero());
-	let genesis_authorities: Option<Vec<AuthorityIdFor<B>>> = cache
+	let genesis_authorities: Option<Vec<A>> = cache
 		.get_at(&well_known_cache_keys::AUTHORITIES, &genesis_id)
 		.and_then(|v| Decode::decode(&mut &v[..]));
 	if genesis_authorities.is_some() {
@@ -642,10 +627,11 @@ fn initialize_authorities_cache<B, C>(client: &C) -> Result<(), ConsensusError>
 }
 
 #[allow(deprecated)]
-fn authorities<B, C>(client: &C, at: &BlockId<B>) -> Result<Vec<AuthorityIdFor<B>>, ConsensusError> where
+fn authorities<A, B, C>(client: &C, at: &BlockId<B>) -> Result<Vec<A>, ConsensusError> where
+	A: Codec,
 	B: Block,
 	C: ProvideRuntimeApi + ProvideCache<B>,
-	C::Api: AuthoritiesApi<B>,
+	C::Api: AuraApi<B, A>,
 {
 	client
 		.cache()
@@ -653,7 +639,7 @@ fn authorities<B, C>(client: &C, at: &BlockId<B>) -> Result<Vec<AuthorityIdFor<B
 			.get_at(&well_known_cache_keys::AUTHORITIES, at)
 			.and_then(|v| Decode::decode(&mut &v[..]))
 		)
-		.or_else(|| AuthoritiesApi::authorities(&*client.runtime_api(), at).ok())
+		.or_else(|| AuraApi::authorities(&*client.runtime_api(), at).ok())
 		.ok_or_else(|| consensus_common::Error::InvalidAuthoritiesSet.into())
 }
 
@@ -687,8 +673,8 @@ pub fn import_queue<B, C, P>(
 ) -> Result<AuraImportQueue<B>, consensus_common::Error> where
 	B: Block,
 	C: 'static + ProvideRuntimeApi + ProvideCache<B> + Send + Sync + AuxStore,
-	C::Api: BlockBuilderApi<B> + AuthoritiesApi<B>,
-	DigestItemFor<B>: CompatibleDigestItem<P> + DigestItem<AuthorityId=AuthorityId<P>>,
+	C::Api: BlockBuilderApi<B> + AuraApi<B, AuthorityId<P>>,
+	DigestItemFor<B>: CompatibleDigestItem<P>,
 	P: Pair + Send + Sync + 'static,
 	P::Public: Clone + Eq + Send + Sync + Hash + Debug + Encode + Decode + AsRef<P::Public>,
 	P::Signature: Encode + Decode,
@@ -730,7 +716,12 @@ mod tests {
 
 	type Error = client::error::Error;
 
-	type TestClient = client::Client<test_client::Backend, test_client::Executor, TestBlock, test_client::runtime::RuntimeApi>;
+	type TestClient = client::Client<
+		test_client::Backend,
+		test_client::Executor,
+		TestBlock,
+		test_client::runtime::RuntimeApi
+	>;
 
 	struct DummyFactory(Arc<TestClient>);
 	struct DummyProposer(u64, Arc<TestClient>);
@@ -739,7 +730,7 @@ mod tests {
 		type Proposer = DummyProposer;
 		type Error = Error;
 
-		fn init(&self, parent_header: &<TestBlock as BlockT>::Header, _authorities: &[AuthorityId<sr25519::Pair>])
+		fn init(&self, parent_header: &<TestBlock as BlockT>::Header)
 			-> Result<DummyProposer, Error>
 		{
 			Ok(DummyProposer(parent_header.number + 1, self.0.clone()))