New sessions, kill consensus module (#2802)

* Draft of new sessions

* Reintroduce tuple impls

* Move staking module to new session API

* More work on staking and grandpa.

* Use iterator to avoid cloning and tuple macro

* Make runtime build again

* Polish the OpaqueKeys devex

* Move consensus logic into system & aura.

* Fix up system module

* Get build mostly going. Stuck at service.rs

* Building again

* Update srml/staking/src/lib.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Refactoring out Consensus module, AuthorityIdOf, &c.

* Refactored out DigestItem::AuthoritiesChanged. Building.

* Remove tentative code

* Remove invalid comment

* Make Seal opaque and introduce nice methods for handling opaque items.

* Start to use proper digest for Aura authorities tracking.

* Fix up grandpa, remove system::Raw/Log

* Refactor Grandpa to use new logging infrastructure.

Also make authorityid/sessionkey static. Switch over to storing
authorities in a straight Vec.

* Building again

* Tidy up some AuthorityIds

* Expunge most of the rest of the AuthorityKey confusion.

Also, de-generify Babe and re-generify Aura.

* Remove cruft

* Untangle last of the `AuthorityId`s.

* Sort out finality_tracker

* Refactor median getting

* Apply suggestions from code review

Co-Authored-By: Robert Habermeier <rphmeier@gmail.com>

* Session tests works

* Update core/sr-primitives/src/generic/digest.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Session tests works

* Fix for staking from @dvc94ch

* log an error

* fix test runtime build

* Some test fixes

* Staking mock update to new session api.

* Fix build.

* Move OpaqueKeys to primitives.

* Use on_initialize instead of check_rotate_session.

* Update tests to new staking api.

* fixup mock

* Fix bond_extra_and_withdraw_unbonded_works.

* Fix bond_with_little_staked_value_bounded_by_slot_stake.

* Fix bond_with_no_staked_value.

* Fix change_controller_works.

* Fix less_than_needed_candidates_works.

* Fix multi_era_reward_should_work.

* Fix nominating_and_rewards_should_work.

* Fix nominators_also_get_slashed.

* Fix phragmen_large_scale_test.

* Fix phragmen_poc_works.

* Fix phragmen_score_should_be_accurate_on_large_stakes.

* Fix phragmen_should_not_overflow.

* Fix reward_destination_works.

* Fix rewards_should_work.

* Fix sessions_and_eras_should_work.

* Fix slot_stake_is_least_staked_validator.

* Fix too_many_unbond_calls_should_not_work.

* Fix wrong_vote_is_null.

* Fix runtime.

* Fix wasm runtime build.

* Update Cargo.lock

* Fix warnings.

* Fix grandpa tests.

* Fix test-runtime build.

* Fix template node build.

* Fix stuff.

* Update Cargo.lock to fix CI

* Re-add missing AuRa logs

Runtimes are required to know about every digest they receive ― they
panic otherwise.  This re-adds support for AuRa pre-runtime digests.

* Update core/consensus/babe/src/digest.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Kill log trait and all that jazz.

* Refactor staking tests.

* Fix ci runtime wasm check.

* Line length 120.

* Make tests build again

* Remove trailing commas in function declarations

The `extern_functions!` macro doesn’t like them, perhaps due to a bug in
rustc.

* Fix type error

* Fix compilation errors

* Fix a test

* Another couple of fixes

* Fix another test

* More test fixes

* Another test fix

* Bump runtime.

* Wrap long line

* Fix build, remove redundant code.

* Issue to track TODO

* Leave the benchmark code alone.

* Fix missing `std::time::{Instant, Duration}`

* Indentation

* Aura ConsensusLog as enum

substrate/core/client/db/src/lib.rs

@@ -46,12 +46,13 @@ use trie::{MemoryDB, PrefixedMemoryDB, prefixed_key};
 use parking_lot::{Mutex, RwLock};
 use primitives::{H256, Blake2Hasher, ChangesTrieConfiguration, convert_hash};
 use primitives::storage::well_known_keys;
-use runtime_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay};
-use runtime_primitives::traits::{
-	Block as BlockT, Header as HeaderT, NumberFor, Zero, One, Digest, DigestItem,
-	SaturatedConversion
+use runtime_primitives::{
+	generic::{BlockId, DigestItem}, Justification, StorageOverlay, ChildrenStorageOverlay,
+	BuildStorage
+};
+use runtime_primitives::traits::{
+	Block as BlockT, Header as HeaderT, NumberFor, Zero, One, SaturatedConversion
 };
-use runtime_primitives::BuildStorage;
 use state_machine::backend::Backend as StateBackend;
 use executor::RuntimeInfo;
 use state_machine::{CodeExecutor, DBValue};
@@ -428,7 +429,11 @@ where Block: BlockT<Hash=H256>,
 		Ok(())
 	}
 
-	fn reset_storage(&mut self, top: StorageOverlay, children: ChildrenStorageOverlay) -> Result<H256, client::error::Error> {
+	fn reset_storage(
+		&mut self,
+		top: StorageOverlay,
+		children: ChildrenStorageOverlay
+	) -> Result<H256, client::error::Error> {
 
 		if top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) {
 			return Err(client::error::Error::GenesisInvalid.into());
@@ -542,7 +547,13 @@ impl<Block: BlockT<Hash=H256>> DbChangesTrieStorage<Block> {
 	}
 
 	/// Prune obsolete changes tries.
-	pub fn prune(&self, config: &ChangesTrieConfiguration, tx: &mut DBTransaction, block_hash: Block::Hash, block_num: NumberFor<Block>) {
+	pub fn prune(
+		&self,
+		config: &ChangesTrieConfiguration,
+		tx: &mut DBTransaction,
+		block_hash: Block::Hash,
+		block_num: NumberFor<Block>,
+	) {
 		// never prune on archive nodes
 		let min_blocks_to_keep = match self.min_blocks_to_keep {
 			Some(min_blocks_to_keep) => min_blocks_to_keep,
@@ -1429,7 +1440,6 @@ mod tests {
 		changes: Vec<(Vec<u8>, Vec<u8>)>,
 		extrinsics_root: H256,
 	) -> H256 {
-		use runtime_primitives::generic::DigestItem;
 		use runtime_primitives::testing::Digest;
 
 		let (changes_root, changes_trie_update) = prepare_changes(changes);

substrate/core/client/db/src/light.rs

@@ -31,11 +31,8 @@ use client::error::{Error as ClientError, Result as ClientResult};
 use client::light::blockchain::Storage as LightBlockchainStorage;
 use parity_codec::{Decode, Encode};
 use primitives::Blake2Hasher;
-use runtime_primitives::generic::BlockId;
-use runtime_primitives::traits::{
-	Block as BlockT, Header as HeaderT,
-	Zero, One, NumberFor, Digest, DigestItem,
-};
+use runtime_primitives::generic::{DigestItem, BlockId};
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, One, NumberFor};
 use consensus_common::well_known_cache_keys;
 use crate::cache::{DbCacheSync, DbCache, ComplexBlockId, EntryType as CacheEntryType};
 use crate::utils::{self, meta_keys, Meta, db_err, read_db, block_id_to_lookup_key, read_meta};
@@ -574,11 +571,10 @@ pub(crate) mod tests {
 	use client::cht;
 	use runtime_primitives::generic::DigestItem;
 	use runtime_primitives::testing::{H256 as Hash, Header, Block as RawBlock, ExtrinsicWrapper};
-	use runtime_primitives::traits::AuthorityIdFor;
 	use super::*;
 
 	type Block = RawBlock<ExtrinsicWrapper<u32>>;
-	type AuthorityId = AuthorityIdFor<Block>;
+	type AuthorityId = primitives::ed25519::Public;
 
 	pub fn default_header(parent: &Hash, number: u64) -> Header {
 		Header {
@@ -871,7 +867,7 @@ pub(crate) mod tests {
 	fn authorities_are_cached() {
 		let db = LightStorage::new_test();
 
-		fn run_checks(db: &LightStorage<Block>, max: u64, checks: &[(u64, Option<Vec<AuthorityIdFor<Block>>>)]) {
+		fn run_checks(db: &LightStorage<Block>, max: u64, checks: &[(u64, Option<Vec<AuthorityId>>)]) {
 			for (at, expected) in checks.iter().take_while(|(at, _)| *at <= max) {
 				let actual = get_authorities(db.cache(), BlockId::Number(*at));
 				assert_eq!(*expected, actual);