New sessions, kill consensus module (#2802)

* Draft of new sessions

* Reintroduce tuple impls

* Move staking module to new session API

* More work on staking and grandpa.

* Use iterator to avoid cloning and tuple macro

* Make runtime build again

* Polish the OpaqueKeys devex

* Move consensus logic into system & aura.

* Fix up system module

* Get build mostly going. Stuck at service.rs

* Building again

* Update srml/staking/src/lib.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Refactoring out Consensus module, AuthorityIdOf, &c.

* Refactored out DigestItem::AuthoritiesChanged. Building.

* Remove tentative code

* Remove invalid comment

* Make Seal opaque and introduce nice methods for handling opaque items.

* Start to use proper digest for Aura authorities tracking.

* Fix up grandpa, remove system::Raw/Log

* Refactor Grandpa to use new logging infrastructure.

Also make authorityid/sessionkey static. Switch over to storing
authorities in a straight Vec.

* Building again

* Tidy up some AuthorityIds

* Expunge most of the rest of the AuthorityKey confusion.

Also, de-generify Babe and re-generify Aura.

* Remove cruft

* Untangle last of the `AuthorityId`s.

* Sort out finality_tracker

* Refactor median getting

* Apply suggestions from code review

Co-Authored-By: Robert Habermeier <rphmeier@gmail.com>

* Session tests works

* Update core/sr-primitives/src/generic/digest.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Session tests works

* Fix for staking from @dvc94ch

* log an error

* fix test runtime build

* Some test fixes

* Staking mock update to new session api.

* Fix build.

* Move OpaqueKeys to primitives.

* Use on_initialize instead of check_rotate_session.

* Update tests to new staking api.

* fixup mock

* Fix bond_extra_and_withdraw_unbonded_works.

* Fix bond_with_little_staked_value_bounded_by_slot_stake.

* Fix bond_with_no_staked_value.

* Fix change_controller_works.

* Fix less_than_needed_candidates_works.

* Fix multi_era_reward_should_work.

* Fix nominating_and_rewards_should_work.

* Fix nominators_also_get_slashed.

* Fix phragmen_large_scale_test.

* Fix phragmen_poc_works.

* Fix phragmen_score_should_be_accurate_on_large_stakes.

* Fix phragmen_should_not_overflow.

* Fix reward_destination_works.

* Fix rewards_should_work.

* Fix sessions_and_eras_should_work.

* Fix slot_stake_is_least_staked_validator.

* Fix too_many_unbond_calls_should_not_work.

* Fix wrong_vote_is_null.

* Fix runtime.

* Fix wasm runtime build.

* Update Cargo.lock

* Fix warnings.

* Fix grandpa tests.

* Fix test-runtime build.

* Fix template node build.

* Fix stuff.

* Update Cargo.lock to fix CI

* Re-add missing AuRa logs

Runtimes are required to know about every digest they receive ― they
panic otherwise.  This re-adds support for AuRa pre-runtime digests.

* Update core/consensus/babe/src/digest.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Kill log trait and all that jazz.

* Refactor staking tests.

* Fix ci runtime wasm check.

* Line length 120.

* Make tests build again

* Remove trailing commas in function declarations

The `extern_functions!` macro doesn’t like them, perhaps due to a bug in
rustc.

* Fix type error

* Fix compilation errors

* Fix a test

* Another couple of fixes

* Fix another test

* More test fixes

* Another test fix

* Bump runtime.

* Wrap long line

* Fix build, remove redundant code.

* Issue to track TODO

* Leave the benchmark code alone.

* Fix missing `std::time::{Instant, Duration}`

* Indentation

* Aura ConsensusLog as enum

substrate/core/finality-grandpa/primitives/Cargo.toml

@@ -10,6 +10,7 @@ substrate-primitives = { path = "../../primitives", default-features = false }
 parity-codec = { version = "3.3", default-features = false, features = ["derive"] }
 sr-primitives = { path = "../../sr-primitives", default-features = false }
 rstd = { package = "sr-std", path = "../../sr-std", default-features = false }
+serde = { version = "1.0", optional = true, features = ["derive"] }
 
 [features]
 default = ["std"]
@@ -19,4 +20,5 @@ std = [
 	"parity-codec/std",
 	"sr-primitives/std",
 	"rstd/std",
+	"serde",
 ]

substrate/core/finality-grandpa/primitives/src/lib.rs

@@ -21,17 +21,32 @@
 #[cfg(not(feature = "std"))]
 extern crate alloc;
 
+#[cfg(feature = "std")]
+use serde::Serialize;
 use parity_codec::{Encode, Decode};
-use substrate_primitives::ed25519;
-use sr_primitives::traits::{DigestFor, NumberFor};
+use sr_primitives::{ConsensusEngineId, traits::{DigestFor, NumberFor}};
 use client::decl_runtime_apis;
 use rstd::vec::Vec;
 
-use ed25519::Public as AuthorityId;
+/// The grandpa crypto scheme defined via the keypair type.
+#[cfg(feature = "std")]
+pub type AuthorityPair = substrate_primitives::ed25519::Pair;
+
+/// Identity of a Grandpa authority.
+pub type AuthorityId = substrate_primitives::ed25519::Public;
+
+/// Signature for a Grandpa authority.
+pub type AuthoritySignature = substrate_primitives::ed25519::Signature;
+
+/// The `ConsensusEngineId` of GRANDPA.
+pub const GRANDPA_ENGINE_ID: ConsensusEngineId = *b"FRNK";
+
+/// The weight of an authority.
+pub type AuthorityWeight = u64;
 
 /// A scheduled change of authority set.
-#[cfg_attr(feature = "std", derive(Debug, PartialEq))]
-#[derive(Clone, Encode, Decode)]
+#[cfg_attr(feature = "std", derive(Debug, Serialize))]
+#[derive(Clone, Eq, PartialEq, Encode, Decode)]
 pub struct ScheduledChange<N> {
 	/// The new authorities after the change, along with their respective weights.
 	pub next_authorities: Vec<(AuthorityId, u64)>,
@@ -44,14 +59,6 @@ pub const PENDING_CHANGE_CALL: &str = "grandpa_pending_change";
 /// WASM function call to get current GRANDPA authorities.
 pub const AUTHORITIES_CALL: &str = "grandpa_authorities";
 
-/// Well-known storage keys for GRANDPA.
-pub mod well_known_keys {
-	/// The key for the authorities and weights vector in storage.
-	pub const AUTHORITY_PREFIX: &[u8] = b":grandpa:auth:";
-	/// The key for the authorities count.
-	pub const AUTHORITY_COUNT: &[u8] = b":grandpa:auth:len";
-}
-
 decl_runtime_apis! {
 	/// APIs for integrating the GRANDPA finality gadget into runtimes.
 	/// This should be implemented on the runtime side.
@@ -107,6 +114,6 @@ decl_runtime_apis! {
 		/// When called at block B, it will return the set of authorities that should be
 		/// used to finalize descendants of this block (B+1, B+2, ...). The block B itself
 		/// is finalized by the authorities from block B-1.
-		fn grandpa_authorities() -> Vec<(AuthorityId, u64)>;
+		fn grandpa_authorities() -> Vec<(AuthorityId, AuthorityWeight)>;
 	}
 }

substrate/core/finality-grandpa/src/authorities.rs

@@ -18,19 +18,17 @@
 
 use fork_tree::ForkTree;
 use parking_lot::RwLock;
-use substrate_primitives::ed25519;
 use grandpa::voter_set::VoterSet;
 use parity_codec::{Encode, Decode};
 use log::{debug, info};
 use substrate_telemetry::{telemetry, CONSENSUS_INFO};
+use fg_primitives::AuthorityId;
 
 use std::cmp::Ord;
 use std::fmt::Debug;
 use std::ops::Add;
 use std::sync::Arc;
 
-use ed25519::Public as AuthorityId;
-
 /// A shared authority set.
 pub(crate) struct SharedAuthoritySet<H, N> {
 	inner: Arc<RwLock<AuthoritySet<H, N>>>,
@@ -522,8 +520,8 @@ mod tests {
 			pending_forced_changes: Vec::new(),
 		};
 
-		let set_a = vec![(AuthorityId([1; 32]), 5)];
-		let set_b = vec![(AuthorityId([2; 32]), 5)];
+		let set_a = vec![(AuthorityId::from_raw([1; 32]), 5)];
+		let set_b = vec![(AuthorityId::from_raw([2; 32]), 5)];
 
 		// two competing changes at the same height on different forks
 		let change_a = PendingChange {
@@ -587,8 +585,8 @@ mod tests {
 			pending_forced_changes: Vec::new(),
 		};
 
-		let set_a = vec![(AuthorityId([1; 32]), 5)];
-		let set_c = vec![(AuthorityId([2; 32]), 5)];
+		let set_a = vec![(AuthorityId::from_raw([1; 32]), 5)];
+		let set_c = vec![(AuthorityId::from_raw([2; 32]), 5)];
 
 		// two competing changes at the same height on different forks
 		let change_a = PendingChange {
@@ -653,7 +651,7 @@ mod tests {
 			pending_forced_changes: Vec::new(),
 		};
 
-		let set_a = vec![(AuthorityId([1; 32]), 5)];
+		let set_a = vec![(AuthorityId::from_raw([1; 32]), 5)];
 
 		let change_a = PendingChange {
 			next_authorities: set_a.clone(),
@@ -719,8 +717,8 @@ mod tests {
 			pending_forced_changes: Vec::new(),
 		};
 
-		let set_a = vec![(AuthorityId([1; 32]), 5)];
-		let set_b = vec![(AuthorityId([2; 32]), 5)];
+		let set_a = vec![(AuthorityId::from_raw([1; 32]), 5)];
+		let set_b = vec![(AuthorityId::from_raw([2; 32]), 5)];
 
 		let change_a = PendingChange {
 			next_authorities: set_a.clone(),

substrate/core/finality-grandpa/src/aux_schema.rs

@@ -26,14 +26,13 @@ use grandpa::round::State as RoundState;
 use runtime_primitives::traits::{Block as BlockT, NumberFor};
 use log::{info, warn};
 use substrate_telemetry::{telemetry, CONSENSUS_INFO};
+use fg_primitives::AuthorityId;
 
 use crate::authorities::{AuthoritySet, SharedAuthoritySet, PendingChange, DelayKind};
 use crate::consensus_changes::{SharedConsensusChanges, ConsensusChanges};
 use crate::environment::{CompletedRound, CompletedRounds, HasVoted, SharedVoterSetState, VoterSetState};
 use crate::NewAuthoritySet;
 
-use substrate_primitives::ed25519::Public as AuthorityId;
-
 const VERSION_KEY: &[u8] = b"grandpa_schema_version";
 const SET_STATE_KEY: &[u8] = b"grandpa_completed_round";
 const AUTHORITY_SET_KEY: &[u8] = b"grandpa_voters";

substrate/core/finality-grandpa/src/communication/mod.rs

@@ -37,7 +37,6 @@ use log::{debug, trace};
 use parity_codec::{Encode, Decode};
 use substrate_primitives::{ed25519, Pair};
 use substrate_telemetry::{telemetry, CONSENSUS_DEBUG, CONSENSUS_INFO};
-use runtime_primitives::ConsensusEngineId;
 use runtime_primitives::traits::{Block as BlockT, Hash as HashT, Header as HeaderT};
 use network::{consensus_gossip as network_gossip, NetworkService};
 use network_gossip::ConsensusMessage;
@@ -55,8 +54,7 @@ mod periodic;
 #[cfg(test)]
 mod tests;
 
-/// The consensus engine ID of GRANDPA.
-pub const GRANDPA_ENGINE_ID: ConsensusEngineId = [b'a', b'f', b'g', b'1'];
+pub use fg_primitives::GRANDPA_ENGINE_ID;
 
 // cost scalars for reporting peers.
 mod cost {

substrate/core/finality-grandpa/src/environment.rs

@@ -50,8 +50,7 @@ use crate::authorities::{AuthoritySet, SharedAuthoritySet};
 use crate::consensus_changes::SharedConsensusChanges;
 use crate::justification::GrandpaJustification;
 use crate::until_imported::UntilVoteTargetImported;
-
-use ed25519::Public as AuthorityId;
+use fg_primitives::AuthorityId;
 
 /// Data about a completed round.
 #[derive(Debug, Clone, Decode, Encode, PartialEq)]

substrate/core/finality-grandpa/src/finality_proof.rs

@@ -49,9 +49,9 @@ use runtime_primitives::{Justification, generic::BlockId};
 use runtime_primitives::traits::{
 	NumberFor, Block as BlockT, Header as HeaderT, One,
 };
-use substrate_primitives::{ed25519, H256, Blake2Hasher};
-use ed25519::Public as AuthorityId;
+use substrate_primitives::{H256, Blake2Hasher};
 use substrate_telemetry::{telemetry, CONSENSUS_INFO};
+use fg_primitives::AuthorityId;
 
 use crate::justification::GrandpaJustification;
 

substrate/core/finality-grandpa/src/justification.rs

@@ -25,13 +25,12 @@ use grandpa::voter_set::VoterSet;
 use grandpa::{Error as GrandpaError};
 use runtime_primitives::generic::BlockId;
 use runtime_primitives::traits::{NumberFor, Block as BlockT, Header as HeaderT};
-use substrate_primitives::{H256, ed25519, Blake2Hasher};
+use substrate_primitives::{H256, Blake2Hasher};
+use fg_primitives::AuthorityId;
 
 use crate::{Commit, Error};
 use crate::communication;
 
-use ed25519::Public as AuthorityId;
-
 /// A GRANDPA justification for block finality, it includes a commit message and
 /// an ancestry proof including all headers routing all precommit target blocks
 /// to the commit target block. Due to the current voting strategy the precommit

substrate/core/finality-grandpa/src/lib.rs

@@ -82,8 +82,6 @@ use std::fmt;
 use std::sync::Arc;
 use std::time::Duration;
 
-pub use fg_primitives::ScheduledChange;
-
 mod authorities;
 mod aux_schema;
 mod communication;
@@ -111,8 +109,10 @@ use import::GrandpaBlockImport;
 use until_imported::UntilCommitBlocksImported;
 use communication::NetworkBridge;
 use service::TelemetryOnConnect;
+use fg_primitives::AuthoritySignature;
 
-use ed25519::{Public as AuthorityId, Signature as AuthoritySignature};
+// Re-export these two because it's just so damn convenient.
+pub use fg_primitives::{AuthorityId, ScheduledChange};
 
 #[cfg(test)]
 mod tests;

substrate/core/finality-grandpa/src/light_import.rs

@@ -35,9 +35,9 @@ use runtime_primitives::Justification;
 use runtime_primitives::traits::{
 	NumberFor, Block as BlockT, Header as HeaderT, ProvideRuntimeApi, DigestFor,
 };
-use fg_primitives::GrandpaApi;
+use fg_primitives::{GrandpaApi, AuthorityId};
 use runtime_primitives::generic::BlockId;
-use substrate_primitives::{H256, Blake2Hasher, ed25519::Public as AuthorityId};
+use substrate_primitives::{H256, Blake2Hasher};
 
 use crate::aux_schema::load_decode;
 use crate::consensus_changes::ConsensusChanges;
@@ -610,7 +610,7 @@ pub mod tests {
 		let client = test_client::new_light();
 		let mut import_data = LightImportData {
 			last_finalized: Default::default(),
-			authority_set: LightAuthoritySet::genesis(vec![(AuthorityId([1; 32]), 1)]),
+			authority_set: LightAuthoritySet::genesis(vec![(AuthorityId::from_raw([1; 32]), 1)]),
 			consensus_changes: ConsensusChanges::empty(),
 		};
 		let block = ImportBlock {
@@ -661,7 +661,7 @@ pub mod tests {
 	#[test]
 	fn finality_proof_required_when_consensus_data_changes_and_no_justification_provided() {
 		let mut cache = HashMap::new();
-		cache.insert(well_known_cache_keys::AUTHORITIES, vec![AuthorityId([2; 32])].encode());
+		cache.insert(well_known_cache_keys::AUTHORITIES, vec![AuthorityId::from_raw([2; 32])].encode());
 		assert_eq!(import_block(cache, None), ImportResult::Imported(ImportedAux {
 			clear_justification_requests: false,
 			needs_justification: false,
@@ -674,7 +674,7 @@ pub mod tests {
 	fn finality_proof_required_when_consensus_data_changes_and_incorrect_justification_provided() {
 		let justification = TestJustification(false, Vec::new()).encode();
 		let mut cache = HashMap::new();
-		cache.insert(well_known_cache_keys::AUTHORITIES, vec![AuthorityId([2; 32])].encode());
+		cache.insert(well_known_cache_keys::AUTHORITIES, vec![AuthorityId::from_raw([2; 32])].encode());
 		assert_eq!(
 			import_block(cache, Some(justification)),
 			ImportResult::Imported(ImportedAux {
@@ -690,7 +690,7 @@ pub mod tests {
 	#[test]
 	fn aux_data_updated_on_start() {
 		let aux_store = InMemoryAuxStore::<Block>::new();
-		let api = Arc::new(TestApi::new(vec![(AuthorityId([1; 32]), 1)]));
+		let api = Arc::new(TestApi::new(vec![(AuthorityId::from_raw([1; 32]), 1)]));
 
 		// when aux store is empty initially
 		assert!(aux_store.get_aux(LIGHT_AUTHORITY_SET_KEY).unwrap().is_none());
@@ -705,7 +705,7 @@ pub mod tests {
 	#[test]
 	fn aux_data_loaded_on_restart() {
 		let aux_store = InMemoryAuxStore::<Block>::new();
-		let api = Arc::new(TestApi::new(vec![(AuthorityId([1; 32]), 1)]));
+		let api = Arc::new(TestApi::new(vec![(AuthorityId::from_raw([1; 32]), 1)]));
 
 		// when aux store is non-empty initially
 		let mut consensus_changes = ConsensusChanges::<H256, u64>::empty();
@@ -714,7 +714,7 @@ pub mod tests {
 			&[
 				(
 					LIGHT_AUTHORITY_SET_KEY,
-					LightAuthoritySet::genesis(vec![(AuthorityId([42; 32]), 2)]).encode().as_slice(),
+					LightAuthoritySet::genesis(vec![(AuthorityId::from_raw([42; 32]), 2)]).encode().as_slice(),
 				),
 				(
 					LIGHT_CONSENSUS_CHANGES_KEY,
@@ -726,7 +726,7 @@ pub mod tests {
 
 		// importer uses it on start
 		let data = load_aux_import_data(Default::default(), &aux_store, api).unwrap();
-		assert_eq!(data.authority_set.authorities(), vec![(AuthorityId([42; 32]), 2)]);
+		assert_eq!(data.authority_set.authorities(), vec![(AuthorityId::from_raw([42; 32]), 2)]);
 		assert_eq!(data.consensus_changes.pending_changes(), &[(42, Default::default())]);
 	}
 }

substrate/core/finality-grandpa/src/observer.rs

@@ -27,7 +27,7 @@ use log::{debug, info, warn};
 use consensus_common::SelectChain;
 use client::{CallExecutor, Client, backend::Backend};
 use runtime_primitives::traits::{NumberFor, Block as BlockT};
-use substrate_primitives::{ed25519::Public as AuthorityId, H256, Blake2Hasher};
+use substrate_primitives::{H256, Blake2Hasher};
 
 use crate::{
 	AuthoritySignature, global_communication, CommandOrError, Config, environment,
@@ -37,6 +37,7 @@ use crate::authorities::SharedAuthoritySet;
 use crate::communication::NetworkBridge;
 use crate::consensus_changes::SharedConsensusChanges;
 use crate::environment::{CompletedRound, CompletedRounds, HasVoted};
+use fg_primitives::AuthorityId;
 
 struct ObserverChain<'a, Block: BlockT, B, E, RA>(&'a Client<B, E, Block, RA>);
 

substrate/core/finality-grandpa/src/tests.rs

@@ -39,7 +39,8 @@ use std::result;
 use parity_codec::Decode;
 use runtime_primitives::traits::{ApiRef, ProvideRuntimeApi, Header as HeaderT};
 use runtime_primitives::generic::BlockId;
-use substrate_primitives::{NativeOrEncoded, ExecutionContext, ed25519::Public as AuthorityId};
+use substrate_primitives::{NativeOrEncoded, ExecutionContext};
+use fg_primitives::AuthorityId;
 
 use authorities::AuthoritySet;
 use finality_proof::{FinalityProofProvider, AuthoritySetForFinalityProver, AuthoritySetForFinalityChecker};
@@ -443,7 +444,7 @@ const TEST_ROUTING_INTERVAL: Duration = Duration::from_millis(50);
 
 fn make_ids(keys: &[AuthorityKeyring]) -> Vec<(substrate_primitives::ed25519::Public, u64)> {
 	keys.iter()
-		.map(|key| AuthorityId(key.to_raw_public()))
+		.map(|key| AuthorityId::from_raw(key.to_raw_public()))
 		.map(|id| (id, 1))
 		.collect()
 }

substrate/core/finality-grandpa/src/until_imported.rs

@@ -28,12 +28,12 @@ use futures::prelude::*;
 use futures::stream::Fuse;
 use parking_lot::Mutex;
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor};
-use substrate_primitives::ed25519::Public as AuthorityId;
 use tokio::timer::Interval;
 
 use std::collections::{HashMap, VecDeque};
 use std::sync::{atomic::{AtomicUsize, Ordering}, Arc};
 use std::time::{Duration, Instant};
+use fg_primitives::AuthorityId;
 
 const LOG_PENDING_INTERVAL: Duration = Duration::from_secs(15);