New sessions, kill consensus module (#2802)

* Draft of new sessions

* Reintroduce tuple impls

* Move staking module to new session API

* More work on staking and grandpa.

* Use iterator to avoid cloning and tuple macro

* Make runtime build again

* Polish the OpaqueKeys devex

* Move consensus logic into system & aura.

* Fix up system module

* Get build mostly going. Stuck at service.rs

* Building again

* Update srml/staking/src/lib.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Refactoring out Consensus module, AuthorityIdOf, &c.

* Refactored out DigestItem::AuthoritiesChanged. Building.

* Remove tentative code

* Remove invalid comment

* Make Seal opaque and introduce nice methods for handling opaque items.

* Start to use proper digest for Aura authorities tracking.

* Fix up grandpa, remove system::Raw/Log

* Refactor Grandpa to use new logging infrastructure.

Also make authorityid/sessionkey static. Switch over to storing
authorities in a straight Vec.

* Building again

* Tidy up some AuthorityIds

* Expunge most of the rest of the AuthorityKey confusion.

Also, de-generify Babe and re-generify Aura.

* Remove cruft

* Untangle last of the `AuthorityId`s.

* Sort out finality_tracker

* Refactor median getting

* Apply suggestions from code review

Co-Authored-By: Robert Habermeier <rphmeier@gmail.com>

* Session tests works

* Update core/sr-primitives/src/generic/digest.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Session tests works

* Fix for staking from @dvc94ch

* log an error

* fix test runtime build

* Some test fixes

* Staking mock update to new session api.

* Fix build.

* Move OpaqueKeys to primitives.

* Use on_initialize instead of check_rotate_session.

* Update tests to new staking api.

* fixup mock

* Fix bond_extra_and_withdraw_unbonded_works.

* Fix bond_with_little_staked_value_bounded_by_slot_stake.

* Fix bond_with_no_staked_value.

* Fix change_controller_works.

* Fix less_than_needed_candidates_works.

* Fix multi_era_reward_should_work.

* Fix nominating_and_rewards_should_work.

* Fix nominators_also_get_slashed.

* Fix phragmen_large_scale_test.

* Fix phragmen_poc_works.

* Fix phragmen_score_should_be_accurate_on_large_stakes.

* Fix phragmen_should_not_overflow.

* Fix reward_destination_works.

* Fix rewards_should_work.

* Fix sessions_and_eras_should_work.

* Fix slot_stake_is_least_staked_validator.

* Fix too_many_unbond_calls_should_not_work.

* Fix wrong_vote_is_null.

* Fix runtime.

* Fix wasm runtime build.

* Update Cargo.lock

* Fix warnings.

* Fix grandpa tests.

* Fix test-runtime build.

* Fix template node build.

* Fix stuff.

* Update Cargo.lock to fix CI

* Re-add missing AuRa logs

Runtimes are required to know about every digest they receive ― they
panic otherwise.  This re-adds support for AuRa pre-runtime digests.

* Update core/consensus/babe/src/digest.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Kill log trait and all that jazz.

* Refactor staking tests.

* Fix ci runtime wasm check.

* Line length 120.

* Make tests build again

* Remove trailing commas in function declarations

The `extern_functions!` macro doesn’t like them, perhaps due to a bug in
rustc.

* Fix type error

* Fix compilation errors

* Fix a test

* Another couple of fixes

* Fix another test

* More test fixes

* Another test fix

* Bump runtime.

* Wrap long line

* Fix build, remove redundant code.

* Issue to track TODO

* Leave the benchmark code alone.

* Fix missing `std::time::{Instant, Duration}`

* Indentation

* Aura ConsensusLog as enum

substrate/srml/executive/src/lib.rs

@@ -77,17 +77,17 @@
 use rstd::prelude::*;
 use rstd::marker::PhantomData;
 use rstd::result;
-use primitives::traits::{
+use primitives::{generic::Digest, traits::{
 	self, Header, Zero, One, Checkable, Applyable, CheckEqual, OnFinalize,
-	OnInitialize, Digest, NumberFor, Block as BlockT, OffchainWorker,
-	ValidateUnsigned, DigestItem,
-};
-use primitives::weights::Weighable;
-use primitives::{ApplyOutcome, ApplyError};
-use primitives::transaction_validity::{TransactionValidity, TransactionPriority, TransactionLongevity};
+	OnInitialize, NumberFor, Block as BlockT, OffchainWorker,
+	ValidateUnsigned,
+}};
 use srml_support::{Dispatchable, traits::MakePayment};
 use parity_codec::{Codec, Encode};
-use system::extrinsics_root;
+use system::{extrinsics_root, DigestOf};
+use primitives::{ApplyOutcome, ApplyError};
+use primitives::transaction_validity::{TransactionValidity, TransactionPriority, TransactionLongevity};
+use primitives::weights::Weighable;
 
 mod internal {
 	pub const MAX_TRANSACTIONS_WEIGHT: u32 = 4 * 1024 * 1024;
@@ -112,6 +112,10 @@ pub trait ExecuteBlock<Block: BlockT> {
 	fn execute_block(block: Block);
 }
 
+pub type CheckedOf<E, C> = <E as Checkable<C>>::Checked;
+pub type CallOf<E, C> = <CheckedOf<E, C> as Applyable>::Call;
+pub type OriginOf<E, C> = <CallOf<E, C> as Dispatchable>::Origin;
+
 pub struct Executive<System, Block, Context, Payment, UnsignedValidator, AllModules>(
 	PhantomData<(System, Block, Context, Payment, UnsignedValidator, AllModules)>
 );
@@ -126,12 +130,10 @@ impl<
 > ExecuteBlock<Block> for Executive<System, Block, Context, Payment, UnsignedValidator, AllModules>
 where
 	Block::Extrinsic: Checkable<Context> + Codec,
-	<Block::Extrinsic as Checkable<Context>>::Checked:
-		Applyable<Index=System::Index, AccountId=System::AccountId> + Weighable,
-	<<Block::Extrinsic as Checkable<Context>>::Checked as Applyable>::Call: Dispatchable,
-	<<<Block::Extrinsic as Checkable<Context>>::Checked as Applyable>::Call as Dispatchable>::Origin:
-		From<Option<System::AccountId>>,
-	UnsignedValidator: ValidateUnsigned<Call=<<Block::Extrinsic as Checkable<Context>>::Checked as Applyable>::Call>
+	CheckedOf<Block::Extrinsic, Context>: Applyable<Index=System::Index, AccountId=System::AccountId> + Weighable,
+	CallOf<Block::Extrinsic, Context>: Dispatchable,
+	OriginOf<Block::Extrinsic, Context>: From<Option<System::AccountId>>,
+	UnsignedValidator: ValidateUnsigned<Call=CallOf<Block::Extrinsic, Context>>,
 {
 	fn execute_block(block: Block) {
 		Executive::<System, Block, Context, Payment, UnsignedValidator, AllModules>::execute_block(block);
@@ -148,16 +150,14 @@ impl<
 > Executive<System, Block, Context, Payment, UnsignedValidator, AllModules>
 where
 	Block::Extrinsic: Checkable<Context> + Codec,
-	<Block::Extrinsic as Checkable<Context>>::Checked:
-		Applyable<Index=System::Index, AccountId=System::AccountId> + Weighable,
-	<<Block::Extrinsic as Checkable<Context>>::Checked as Applyable>::Call: Dispatchable,
-	<<<Block::Extrinsic as Checkable<Context>>::Checked as Applyable>::Call as Dispatchable>::Origin:
-		From<Option<System::AccountId>>,
-	UnsignedValidator: ValidateUnsigned<Call=<<Block::Extrinsic as Checkable<Context>>::Checked as Applyable>::Call>
+	CheckedOf<Block::Extrinsic, Context>: Applyable<Index=System::Index, AccountId=System::AccountId> + Weighable,
+	CallOf<Block::Extrinsic, Context>: Dispatchable,
+	OriginOf<Block::Extrinsic, Context>: From<Option<System::AccountId>>,
+	UnsignedValidator: ValidateUnsigned<Call=CallOf<Block::Extrinsic, Context>>,
 {
 	/// Start the execution of a particular block.
 	pub fn initialize_block(header: &System::Header) {
-		let mut digests = System::Digest::default();
+		let mut digests = <DigestOf<System>>::default();
 		header.digest().logs().iter().for_each(|d| if d.as_pre_runtime().is_some() { digests.push(d.clone()) });
 		Self::initialize_block_impl(header.number(), header.parent_hash(), header.extrinsics_root(), &digests);
 	}
@@ -166,7 +166,7 @@ where
 		block_number: &System::BlockNumber,
 		parent_hash: &System::Hash,
 		extrinsics_root: &System::Hash,
-		digest: &System::Digest
+		digest: &Digest<System::Hash>,
 	) {
 		<system::Module<System>>::initialize(block_number, parent_hash, extrinsics_root, digest);
 		<AllModules as OnInitialize<System::BlockNumber>>::on_initialize(*block_number);
@@ -259,7 +259,7 @@ where
 	fn apply_extrinsic_with_len(
 		uxt: Block::Extrinsic,
 		encoded_len: usize,
-		to_note: Option<Vec<u8>>
+		to_note: Option<Vec<u8>>,
 	) -> result::Result<internal::ApplyOutcome, internal::ApplyError> {
 		// Verify that the signature is good.
 		let xt = uxt.check(&Default::default()).map_err(internal::ApplyError::BadSignature)?;
@@ -397,7 +397,7 @@ mod tests {
 	use substrate_primitives::{H256, Blake2Hasher};
 	use primitives::BuildStorage;
 	use primitives::traits::{Header as HeaderT, BlakeTwo256, IdentityLookup};
-	use primitives::testing::{Digest, DigestItem, Header, Block};
+	use primitives::testing::{Digest, Header, Block};
 	use srml_support::{traits::Currency, impl_outer_origin, impl_outer_event};
 	use system;
 	use hex_literal::hex;
@@ -422,12 +422,10 @@ mod tests {
 		type BlockNumber = u64;
 		type Hash = substrate_primitives::H256;
 		type Hashing = BlakeTwo256;
-		type Digest = Digest;
 		type AccountId = u64;
 		type Lookup = IdentityLookup<u64>;
 		type Header = Header;
 		type Event = MetaEvent;
-		type Log = DigestItem;
 	}
 	impl balances::Trait for Runtime {
 		type Balance = u64;
@@ -457,7 +455,14 @@ mod tests {
 	}
 
 	type TestXt = primitives::testing::TestXt<Call<Runtime>>;
-	type Executive = super::Executive<Runtime, Block<TestXt>, system::ChainContext<Runtime>, balances::Module<Runtime>, Runtime, ()>;
+	type Executive = super::Executive<
+		Runtime,
+		Block<TestXt>,
+		system::ChainContext<Runtime>,
+		balances::Module<Runtime>,
+		Runtime,
+		()
+	>;
 
 	#[test]
 	fn balance_transfer_dispatch_works() {
@@ -474,8 +479,13 @@ mod tests {
 		let xt = primitives::testing::TestXt(Some(1), 0, Call::transfer(2, 69));
 		let mut t = runtime_io::TestExternalities::<Blake2Hasher>::new(t);
 		with_externalities(&mut t, || {
-			Executive::initialize_block(&Header::new(1, H256::default(), H256::default(),
-				[69u8; 32].into(), Digest::default()));
+			Executive::initialize_block(&Header::new(
+				1,
+				H256::default(),
+				H256::default(),
+				[69u8; 32].into(),
+				Digest::default(),
+			));
 			Executive::apply_extrinsic(xt).unwrap();
 			assert_eq!(<balances::Module<Runtime>>::total_balance(&1), 32);
 			assert_eq!(<balances::Module<Runtime>>::total_balance(&2), 69);
@@ -543,7 +553,13 @@ mod tests {
 		let mut t = new_test_ext();
 		let xt = primitives::testing::TestXt(Some(1), 42, Call::transfer(33, 69));
 		with_externalities(&mut t, || {
-			Executive::initialize_block(&Header::new(1, H256::default(), H256::default(), [69u8; 32].into(), Digest::default()));
+			Executive::initialize_block(&Header::new(
+				1,
+				H256::default(),
+				H256::default(),
+				[69u8; 32].into(),
+				Digest::default(),
+			));
 			assert!(Executive::apply_extrinsic(xt).is_err());
 			assert_eq!(<system::Module<Runtime>>::extrinsic_index(), Some(0));
 		});
@@ -558,7 +574,13 @@ mod tests {
 			let encoded = xt2.encode();
 			let len = if should_fail { (internal::MAX_TRANSACTIONS_WEIGHT - 1) as usize } else { encoded.len() };
 			with_externalities(&mut t, || {
-				Executive::initialize_block(&Header::new(1, H256::default(), H256::default(), [69u8; 32].into(), Digest::default()));
+				Executive::initialize_block(&Header::new(
+					1,
+					H256::default(),
+					H256::default(),
+					[69u8; 32].into(),
+					Digest::default(),
+				));
 				assert_eq!(<system::Module<Runtime>>::all_extrinsics_weight(), 0);
 
 				Executive::apply_extrinsic(xt).unwrap();