New sessions, kill consensus module (#2802)

* Draft of new sessions

* Reintroduce tuple impls

* Move staking module to new session API

* More work on staking and grandpa.

* Use iterator to avoid cloning and tuple macro

* Make runtime build again

* Polish the OpaqueKeys devex

* Move consensus logic into system & aura.

* Fix up system module

* Get build mostly going. Stuck at service.rs

* Building again

* Update srml/staking/src/lib.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Refactoring out Consensus module, AuthorityIdOf, &c.

* Refactored out DigestItem::AuthoritiesChanged. Building.

* Remove tentative code

* Remove invalid comment

* Make Seal opaque and introduce nice methods for handling opaque items.

* Start to use proper digest for Aura authorities tracking.

* Fix up grandpa, remove system::Raw/Log

* Refactor Grandpa to use new logging infrastructure.

Also make authorityid/sessionkey static. Switch over to storing
authorities in a straight Vec.

* Building again

* Tidy up some AuthorityIds

* Expunge most of the rest of the AuthorityKey confusion.

Also, de-generify Babe and re-generify Aura.

* Remove cruft

* Untangle last of the `AuthorityId`s.

* Sort out finality_tracker

* Refactor median getting

* Apply suggestions from code review

Co-Authored-By: Robert Habermeier <rphmeier@gmail.com>

* Session tests works

* Update core/sr-primitives/src/generic/digest.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Session tests works

* Fix for staking from @dvc94ch

* log an error

* fix test runtime build

* Some test fixes

* Staking mock update to new session api.

* Fix build.

* Move OpaqueKeys to primitives.

* Use on_initialize instead of check_rotate_session.

* Update tests to new staking api.

* fixup mock

* Fix bond_extra_and_withdraw_unbonded_works.

* Fix bond_with_little_staked_value_bounded_by_slot_stake.

* Fix bond_with_no_staked_value.

* Fix change_controller_works.

* Fix less_than_needed_candidates_works.

* Fix multi_era_reward_should_work.

* Fix nominating_and_rewards_should_work.

* Fix nominators_also_get_slashed.

* Fix phragmen_large_scale_test.

* Fix phragmen_poc_works.

* Fix phragmen_score_should_be_accurate_on_large_stakes.

* Fix phragmen_should_not_overflow.

* Fix reward_destination_works.

* Fix rewards_should_work.

* Fix sessions_and_eras_should_work.

* Fix slot_stake_is_least_staked_validator.

* Fix too_many_unbond_calls_should_not_work.

* Fix wrong_vote_is_null.

* Fix runtime.

* Fix wasm runtime build.

* Update Cargo.lock

* Fix warnings.

* Fix grandpa tests.

* Fix test-runtime build.

* Fix template node build.

* Fix stuff.

* Update Cargo.lock to fix CI

* Re-add missing AuRa logs

Runtimes are required to know about every digest they receive ― they
panic otherwise.  This re-adds support for AuRa pre-runtime digests.

* Update core/consensus/babe/src/digest.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Kill log trait and all that jazz.

* Refactor staking tests.

* Fix ci runtime wasm check.

* Line length 120.

* Make tests build again

* Remove trailing commas in function declarations

The `extern_functions!` macro doesn’t like them, perhaps due to a bug in
rustc.

* Fix type error

* Fix compilation errors

* Fix a test

* Another couple of fixes

* Fix another test

* More test fixes

* Another test fix

* Bump runtime.

* Wrap long line

* Fix build, remove redundant code.

* Issue to track TODO

* Leave the benchmark code alone.

* Fix missing `std::time::{Instant, Duration}`

* Indentation

* Aura ConsensusLog as enum

substrate/node/runtime/Cargo.toml

@@ -17,7 +17,6 @@ version = { package = "sr-version", path = "../../core/sr-version", default-feat
 support = { package = "srml-support", path = "../../srml/support", default-features = false }
 aura = { package = "srml-aura", path = "../../srml/aura", default-features = false }
 balances = { package = "srml-balances", path = "../../srml/balances", default-features = false }
-consensus = { package = "srml-consensus", path = "../../srml/consensus", default-features = false }
 contract = { package = "srml-contract", path = "../../srml/contract", default-features = false }
 council = { package = "srml-council", path = "../../srml/council", default-features = false }
 democracy = { package = "srml-democracy", path = "../../srml/democracy", default-features = false }
@@ -36,7 +35,6 @@ consensus_aura = { package = "substrate-consensus-aura-primitives", path = "../.
 rustc-hex = { version = "2.0", optional = true }
 serde = { version = "1.0", optional = true }
 substrate-keyring = { path = "../../core/keyring", optional = true }
-consensus_authorities = { package = "substrate-consensus-authorities", path = "../../core/consensus/authorities", default-features = false }
 
 [features]
 default = ["std"]
@@ -51,7 +49,6 @@ std = [
 	"support/std",
 	"aura/std",
 	"balances/std",
-	"consensus/std",
 	"contract/std",
 	"council/std",
 	"democracy/std",
@@ -74,5 +71,4 @@ std = [
 	"rustc-hex",
 	"substrate-keyring",
 	"offchain-primitives/std",
-	"consensus_authorities/std",
 ]

substrate/node/runtime/src/lib.rs

@@ -21,10 +21,11 @@
 #![recursion_limit="256"]
 
 use rstd::prelude::*;
+use parity_codec::{Encode, Decode};
 use support::{construct_runtime, parameter_types};
 use substrate_primitives::u32_trait::{_1, _2, _3, _4};
 use node_primitives::{
-	AccountId, AccountIndex, Balance, BlockNumber, Hash, Index, AuthorityId, Signature, AuthoritySignature
+	AccountId, AccountIndex, Balance, BlockNumber, Hash, Index, Signature, AuraId
 };
 use grandpa::fg_primitives::{self, ScheduledChange};
 use client::{
@@ -34,7 +35,7 @@ use client::{
 use runtime_primitives::{ApplyResult, generic, create_runtime_str};
 use runtime_primitives::transaction_validity::TransactionValidity;
 use runtime_primitives::traits::{
-	BlakeTwo256, Block as BlockT, DigestFor, NumberFor, StaticLookup, AuthorityIdFor, Convert,
+	BlakeTwo256, Block as BlockT, DigestFor, NumberFor, StaticLookup, Convert,
 };
 use version::RuntimeVersion;
 use council::{motions as council_motions};
@@ -43,13 +44,13 @@ use council::seats as council_seats;
 #[cfg(any(feature = "std", test))]
 use version::NativeVersion;
 use substrate_primitives::OpaqueMetadata;
+use grandpa::{AuthorityId as GrandpaId, AuthorityWeight as GrandpaWeight};
 
 #[cfg(any(feature = "std", test))]
 pub use runtime_primitives::BuildStorage;
-pub use consensus::Call as ConsensusCall;
 pub use timestamp::Call as TimestampCall;
 pub use balances::Call as BalancesCall;
-pub use runtime_primitives::{Permill, Perbill};
+pub use runtime_primitives::{Permill, Perbill, impl_opaque_keys};
 pub use support::StorageValue;
 pub use staking::StakerStatus;
 
@@ -58,7 +59,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
 	spec_name: create_runtime_str!("node"),
 	impl_name: create_runtime_str!("substrate-node"),
 	authoring_version: 10,
-	spec_version: 93,
+	spec_version: 94,
 	impl_version: 94,
 	apis: RUNTIME_API_VERSIONS,
 };
@@ -92,16 +93,15 @@ impl system::Trait for Runtime {
 	type BlockNumber = BlockNumber;
 	type Hash = Hash;
 	type Hashing = BlakeTwo256;
-	type Digest = generic::Digest<Log>;
 	type AccountId = AccountId;
 	type Lookup = Indices;
-	type Header = generic::Header<BlockNumber, BlakeTwo256, Log>;
+	type Header = generic::Header<BlockNumber, BlakeTwo256>;
 	type Event = Event;
-	type Log = Log;
 }
 
 impl aura::Trait for Runtime {
 	type HandleReport = aura::StakingSlasher<Runtime>;
+	type AuthorityId = AuraId;
 }
 
 impl indices::Trait for Runtime {
@@ -121,24 +121,40 @@ impl balances::Trait for Runtime {
 	type TransferPayment = ();
 }
 
-impl consensus::Trait for Runtime {
-	type Log = Log;
-	type SessionKey = AuthorityId;
-
-	// The Aura module handles offline-reports internally
-	// rather than using an explicit report system.
-	type InherentOfflineReport = ();
-}
-
 impl timestamp::Trait for Runtime {
 	type Moment = u64;
 	type OnTimestampSet = Aura;
 }
 
+parameter_types! {
+	pub const Period: BlockNumber = 10 * MINUTES;
+	pub const Offset: BlockNumber = 0;
+}
+
+type SessionHandlers = (Grandpa, Aura);
+#[cfg(feature = "std")]
+use serde::{Serialize, Deserialize};
+impl_opaque_keys! {
+	pub struct SessionKeys(grandpa::AuthorityId, AuraId);
+}
+
+// NOTE: `SessionHandler` and `SessionKeys` are co-dependent: One key will be used for each handler.
+// The number and order of items in `SessionHandler` *MUST* be the same number and order of keys in
+// `SessionKeys`.
+// TODO: Introduce some structure to tie these together to make it a bit less of a footgun. This
+// should be easy, since OneSessionHandler trait provides the `Key` as an associated type. #2858
+
 impl session::Trait for Runtime {
-	type ConvertAccountIdToSessionKey = ();
-	type OnSessionChange = (Staking, grandpa::SyncedAuthorities<Runtime>);
+	type OnSessionEnding = Staking;
+	type SessionHandler = SessionHandlers;
+	type ShouldEndSession = session::PeriodicSessions<Period, Offset>;
 	type Event = Event;
+	type Keys = SessionKeys;
+}
+
+parameter_types! {
+	pub const SessionsPerEra: session::SessionIndex = 6;
+	pub const BondingDuration: staking::EraIndex = 24 * 28;
 }
 
 impl staking::Trait for Runtime {
@@ -148,6 +164,8 @@ impl staking::Trait for Runtime {
 	type Event = Event;
 	type Slash = ();
 	type Reward = ();
+	type SessionsPerEra = SessionsPerEra;
+	type BondingDuration = BondingDuration;
 }
 
 const MINUTES: BlockNumber = 6;
@@ -187,7 +205,6 @@ impl council::Trait for Runtime {
 	type OnMembersChanged = CouncilMotions;
 }
 
-
 impl council::motions::Trait for Runtime {
 	type Origin = Origin;
 	type Proposal = Call;
@@ -220,35 +237,32 @@ impl sudo::Trait for Runtime {
 }
 
 impl grandpa::Trait for Runtime {
-	type SessionKey = AuthorityId;
-	type Log = Log;
 	type Event = Event;
 }
 
 impl finality_tracker::Trait for Runtime {
-	type OnFinalizationStalled = grandpa::SyncedAuthorities<Runtime>;
+	type OnFinalizationStalled = Grandpa;
 }
 
 construct_runtime!(
-	pub enum Runtime with Log(InternalLog: DigestItem<Hash, AuthorityId, AuthoritySignature>) where
+	pub enum Runtime where
 		Block = Block,
 		NodeBlock = node_primitives::Block,
 		UncheckedExtrinsic = UncheckedExtrinsic
 	{
-		System: system::{default, Log(ChangesTrieRoot)},
-		Aura: aura::{Module, Inherent(Timestamp), Log(PreRuntime)},
+		System: system::{Module, Call, Storage, Config<T>},
+		Aura: aura::{Module, Config<T>, Inherent(Timestamp)},
 		Timestamp: timestamp::{Module, Call, Storage, Config<T>, Inherent},
-		Consensus: consensus::{Module, Call, Storage, Config<T>, Log(AuthoritiesChange), Inherent},
 		Indices: indices,
 		Balances: balances,
-		Session: session,
+		Session: session::{Module, Call, Storage, Event, Config<T>},
 		Staking: staking::{default, OfflineWorker},
 		Democracy: democracy,
 		Council: council::{Module, Call, Storage, Event<T>},
 		CouncilMotions: council_motions::{Module, Call, Storage, Event<T>, Origin<T>},
 		CouncilSeats: council_seats::{Config<T>},
 		FinalityTracker: finality_tracker::{Module, Call, Inherent},
-		Grandpa: grandpa::{Module, Call, Storage, Config<T>, Log(), Event<T>},
+		Grandpa: grandpa::{Module, Call, Storage, Config<T>, Event},
 		Treasury: treasury,
 		Contract: contract::{Module, Call, Storage, Config<T>, Event<T>},
 		Sudo: sudo,
@@ -258,7 +272,7 @@ construct_runtime!(
 /// The address format for describing accounts.
 pub type Address = <Indices as StaticLookup>::Source;
 /// Block header type as expected by this runtime.
-pub type Header = generic::Header<BlockNumber, BlakeTwo256, Log>;
+pub type Header = generic::Header<BlockNumber, BlakeTwo256>;
 /// Block type as expected by this runtime.
 pub type Block = generic::Block<Header, UncheckedExtrinsic>;
 /// A Block signed with a Justification
@@ -331,45 +345,26 @@ impl_runtime_apis! {
 		fn grandpa_pending_change(digest: &DigestFor<Block>)
 			-> Option<ScheduledChange<NumberFor<Block>>>
 		{
-			for log in digest.logs.iter().filter_map(|l| match l {
-				Log(InternalLog::grandpa(grandpa_signal)) => Some(grandpa_signal),
-				_ => None
-			}) {
-				if let Some(change) = Grandpa::scrape_digest_change(log) {
-					return Some(change);
-				}
-			}
-			None
+			Grandpa::pending_change(digest)
 		}
 
 		fn grandpa_forced_change(digest: &DigestFor<Block>)
 			-> Option<(NumberFor<Block>, ScheduledChange<NumberFor<Block>>)>
 		{
-			for log in digest.logs.iter().filter_map(|l| match l {
-				Log(InternalLog::grandpa(grandpa_signal)) => Some(grandpa_signal),
-				_ => None
-			}) {
-				if let Some(change) = Grandpa::scrape_digest_forced_change(log) {
-					return Some(change);
-				}
-			}
-			None
+			Grandpa::forced_change(digest)
 		}
 
-		fn grandpa_authorities() -> Vec<(AuthorityId, u64)> {
+		fn grandpa_authorities() -> Vec<(GrandpaId, GrandpaWeight)> {
 			Grandpa::grandpa_authorities()
 		}
 	}
 
-	impl consensus_aura::AuraApi<Block> for Runtime {
+	impl consensus_aura::AuraApi<Block, AuraId> for Runtime {
 		fn slot_duration() -> u64 {
 			Aura::slot_duration()
 		}
-	}
-
-	impl consensus_authorities::AuthoritiesApi<Block> for Runtime {
-		fn authorities() -> Vec<AuthorityIdFor<Block>> {
-			Consensus::authorities()
+		fn authorities() -> Vec<AuraId> {
+			Aura::authorities()
 		}
 	}
 }

substrate/node/runtime/wasm/Cargo.lock

@@ -1484,7 +1484,6 @@ dependencies = [
  "sr-version 2.0.0",
  "srml-aura 2.0.0",
  "srml-balances 2.0.0",
- "srml-consensus 2.0.0",
  "srml-contract 2.0.0",
  "srml-council 2.0.0",
  "srml-democracy 2.0.0",
@@ -1501,7 +1500,6 @@ dependencies = [
  "srml-treasury 2.0.0",
  "substrate-client 2.0.0",
  "substrate-consensus-aura-primitives 2.0.0",
- "substrate-consensus-authorities 2.0.0",
  "substrate-keyring 2.0.0",
  "substrate-offchain-primitives 2.0.0",
  "substrate-primitives 2.0.0",
@@ -2425,7 +2423,9 @@ dependencies = [
  "srml-support 2.0.0",
  "srml-system 2.0.0",
  "srml-timestamp 2.0.0",
+ "substrate-consensus-aura-primitives 2.0.0",
  "substrate-inherents 2.0.0",
+ "substrate-primitives 2.0.0",
 ]
 
 [[package]]
@@ -2442,20 +2442,6 @@ dependencies = [
  "substrate-keyring 2.0.0",
 ]
 
-[[package]]
-name = "srml-consensus"
-version = "2.0.0"
-dependencies = [
- "parity-codec 3.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)",
- "sr-primitives 2.0.0",
- "sr-std 2.0.0",
- "srml-support 2.0.0",
- "srml-system 2.0.0",
- "substrate-inherents 2.0.0",
- "substrate-primitives 2.0.0",
-]
-
 [[package]]
 name = "srml-contract"
 version = "2.0.0"
@@ -2539,7 +2525,6 @@ dependencies = [
  "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)",
  "sr-primitives 2.0.0",
  "sr-std 2.0.0",
- "srml-consensus 2.0.0",
  "srml-finality-tracker 2.0.0",
  "srml-session 2.0.0",
  "srml-support 2.0.0",
@@ -2583,7 +2568,6 @@ dependencies = [
  "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)",
  "sr-primitives 2.0.0",
  "sr-std 2.0.0",
- "srml-consensus 2.0.0",
  "srml-support 2.0.0",
  "srml-system 2.0.0",
  "srml-timestamp 2.0.0",
@@ -2599,7 +2583,6 @@ dependencies = [
  "sr-io 2.0.0",
  "sr-primitives 2.0.0",
  "sr-std 2.0.0",
- "srml-consensus 2.0.0",
  "srml-session 2.0.0",
  "srml-support 2.0.0",
  "srml-system 2.0.0",
@@ -2788,21 +2771,10 @@ dependencies = [
 [[package]]
 name = "substrate-consensus-aura-primitives"
 version = "2.0.0"
-dependencies = [
- "sr-primitives 2.0.0",
- "substrate-client 2.0.0",
-]
-
-[[package]]
-name = "substrate-consensus-authorities"
-version = "2.0.0"
 dependencies = [
  "parity-codec 3.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "sr-io 2.0.0",
  "sr-primitives 2.0.0",
  "sr-std 2.0.0",
- "sr-version 2.0.0",
- "srml-support 2.0.0",
  "substrate-client 2.0.0",
  "substrate-primitives 2.0.0",
 ]
@@ -2853,6 +2825,7 @@ name = "substrate-finality-grandpa-primitives"
 version = "2.0.0"
 dependencies = [
  "parity-codec 3.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)",
  "sr-primitives 2.0.0",
  "sr-std 2.0.0",
  "substrate-client 2.0.0",