New sessions, kill consensus module (#2802)

* Draft of new sessions

* Reintroduce tuple impls

* Move staking module to new session API

* More work on staking and grandpa.

* Use iterator to avoid cloning and tuple macro

* Make runtime build again

* Polish the OpaqueKeys devex

* Move consensus logic into system & aura.

* Fix up system module

* Get build mostly going. Stuck at service.rs

* Building again

* Update srml/staking/src/lib.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Refactoring out Consensus module, AuthorityIdOf, &c.

* Refactored out DigestItem::AuthoritiesChanged. Building.

* Remove tentative code

* Remove invalid comment

* Make Seal opaque and introduce nice methods for handling opaque items.

* Start to use proper digest for Aura authorities tracking.

* Fix up grandpa, remove system::Raw/Log

* Refactor Grandpa to use new logging infrastructure.

Also make authorityid/sessionkey static. Switch over to storing
authorities in a straight Vec.

* Building again

* Tidy up some AuthorityIds

* Expunge most of the rest of the AuthorityKey confusion.

Also, de-generify Babe and re-generify Aura.

* Remove cruft

* Untangle last of the `AuthorityId`s.

* Sort out finality_tracker

* Refactor median getting

* Apply suggestions from code review

Co-Authored-By: Robert Habermeier <rphmeier@gmail.com>

* Session tests works

* Update core/sr-primitives/src/generic/digest.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Session tests works

* Fix for staking from @dvc94ch

* log an error

* fix test runtime build

* Some test fixes

* Staking mock update to new session api.

* Fix build.

* Move OpaqueKeys to primitives.

* Use on_initialize instead of check_rotate_session.

* Update tests to new staking api.

* fixup mock

* Fix bond_extra_and_withdraw_unbonded_works.

* Fix bond_with_little_staked_value_bounded_by_slot_stake.

* Fix bond_with_no_staked_value.

* Fix change_controller_works.

* Fix less_than_needed_candidates_works.

* Fix multi_era_reward_should_work.

* Fix nominating_and_rewards_should_work.

* Fix nominators_also_get_slashed.

* Fix phragmen_large_scale_test.

* Fix phragmen_poc_works.

* Fix phragmen_score_should_be_accurate_on_large_stakes.

* Fix phragmen_should_not_overflow.

* Fix reward_destination_works.

* Fix rewards_should_work.

* Fix sessions_and_eras_should_work.

* Fix slot_stake_is_least_staked_validator.

* Fix too_many_unbond_calls_should_not_work.

* Fix wrong_vote_is_null.

* Fix runtime.

* Fix wasm runtime build.

* Update Cargo.lock

* Fix warnings.

* Fix grandpa tests.

* Fix test-runtime build.

* Fix template node build.

* Fix stuff.

* Update Cargo.lock to fix CI

* Re-add missing AuRa logs

Runtimes are required to know about every digest they receive ― they
panic otherwise.  This re-adds support for AuRa pre-runtime digests.

* Update core/consensus/babe/src/digest.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Kill log trait and all that jazz.

* Refactor staking tests.

* Fix ci runtime wasm check.

* Line length 120.

* Make tests build again

* Remove trailing commas in function declarations

The `extern_functions!` macro doesn’t like them, perhaps due to a bug in
rustc.

* Fix type error

* Fix compilation errors

* Fix a test

* Another couple of fixes

* Fix another test

* More test fixes

* Another test fix

* Bump runtime.

* Wrap long line

* Fix build, remove redundant code.

* Issue to track TODO

* Leave the benchmark code alone.

* Fix missing `std::time::{Instant, Duration}`

* Indentation

* Aura ConsensusLog as enum

substrate/core/test-runtime/src/system.rs

@@ -21,13 +21,14 @@ use rstd::prelude::*;
 use runtime_io::{storage_root, enumerated_trie_root, storage_changes_root, twox_128, blake2_256};
 use runtime_support::storage::{self, StorageValue, StorageMap};
 use runtime_support::storage_items;
-use runtime_primitives::traits::{Hash as HashT, BlakeTwo256, Digest as DigestT, Header as _};
+use runtime_primitives::traits::{Hash as HashT, BlakeTwo256, Header as _};
 use runtime_primitives::generic;
 use runtime_primitives::{ApplyError, ApplyOutcome, ApplyResult, transaction_validity::TransactionValidity};
 use parity_codec::{KeyedVec, Encode};
-use super::{AccountId, BlockNumber, Extrinsic, Transfer, H256 as Hash, Block, Header, Digest};
+use super::{
+	AccountId, BlockNumber, Extrinsic, Transfer, H256 as Hash, Block, Header, Digest, AuthorityId
+};
 use primitives::{Blake2Hasher, storage::well_known_keys};
-use primitives::sr25519::Public as AuthorityId;
 
 const NONCE_OF: &[u8] = b"nonce:";
 const BALANCE_OF: &[u8] = b"balance:";
@@ -39,6 +40,7 @@ storage_items! {
 	ParentHash: b"sys:pha" => required Hash;
 	NewAuthorities: b"sys:new_auth" => Vec<AuthorityId>;
 	StorageDigest: b"sys:digest" => Digest;
+	Authorities get(authorities): b"sys:auth" => default Vec<AuthorityId>;
 }
 
 pub fn balance_of_key(who: AccountId) -> Vec<u8> {
@@ -53,17 +55,6 @@ pub fn nonce_of(who: AccountId) -> u64 {
 	storage::hashed::get_or(&blake2_256, &who.to_keyed_vec(NONCE_OF), 0)
 }
 
-/// Get authorities at given block.
-pub fn authorities() -> Vec<AuthorityId> {
-	let len: u32 = storage::unhashed::get(well_known_keys::AUTHORITY_COUNT)
-		.expect("There are always authorities in test-runtime");
-	(0..len)
-		.map(|i| storage::unhashed::get(&i.to_keyed_vec(well_known_keys::AUTHORITY_PREFIX))
-			.expect("Authority is properly encoded in test-runtime")
-		)
-		.collect()
-}
-
 pub fn initialize_block(header: &Header) {
 	// populate environment.
 	<Number>::put(&header.number);
@@ -80,8 +71,25 @@ pub fn take_block_number() -> Option<BlockNumber> {
 	Number::take()
 }
 
+#[derive(Copy, Clone)]
+enum Mode {
+	Verify,
+	Overwrite,
+}
+
 /// Actually execute all transitioning for `block`.
 pub fn polish_block(block: &mut Block) {
+	execute_block_with_state_root_handler(block, Mode::Overwrite);
+}
+
+pub fn execute_block(mut block: Block) {
+	execute_block_with_state_root_handler(&mut block, Mode::Verify);
+}
+
+fn execute_block_with_state_root_handler(
+	block: &mut Block,
+	mode: Mode,
+) {
 	let header = &mut block.header;
 
 	// check transaction trie root represents the transactions.
@@ -89,7 +97,11 @@ pub fn polish_block(block: &mut Block) {
 	let txs = txs.iter().map(Vec::as_slice).collect::<Vec<_>>();
 	let txs_root = enumerated_trie_root::<Blake2Hasher>(&txs).into();
 	info_expect_equal_hash(&txs_root, &header.extrinsics_root);
-	header.extrinsics_root = txs_root;
+	if let Mode::Overwrite = mode {
+		header.extrinsics_root = txs_root;
+	} else {
+		assert!(txs_root == header.extrinsics_root, "Transaction trie root must be valid.");
+	}
 
 	// execute transactions
 	block.extrinsics.iter().enumerate().for_each(|(i, e)| {
@@ -98,7 +110,14 @@ pub fn polish_block(block: &mut Block) {
 		storage::unhashed::kill(well_known_keys::EXTRINSIC_INDEX);
 	});
 
-	header.state_root = storage_root().into();
+	if let Mode::Overwrite = mode {
+		header.state_root = storage_root().into();
+	} else {
+		// check storage root.
+		let storage_root = storage_root().into();
+		info_expect_equal_hash(&storage_root, &header.state_root);
+		assert!(storage_root == header.state_root, "Storage root must match that calculated.");
+	}
 
 	// check digest
 	let digest = &mut header.digest;
@@ -106,39 +125,8 @@ pub fn polish_block(block: &mut Block) {
 		digest.push(generic::DigestItem::ChangesTrieRoot(storage_changes_root.into()));
 	}
 	if let Some(new_authorities) = <NewAuthorities>::take() {
-		digest.push(generic::DigestItem::AuthoritiesChange(new_authorities));
-	}
-}
-
-pub fn execute_block(mut block: Block) {
-	let header = &mut block.header;
-
-	// check transaction trie root represents the transactions.
-	let txs = block.extrinsics.iter().map(Encode::encode).collect::<Vec<_>>();
-	let txs = txs.iter().map(Vec::as_slice).collect::<Vec<_>>();
-	let txs_root = enumerated_trie_root::<Blake2Hasher>(&txs).into();
-	info_expect_equal_hash(&txs_root, &header.extrinsics_root);
-	assert!(txs_root == header.extrinsics_root, "Transaction trie root must be valid.");
-
-	// execute transactions
-	block.extrinsics.into_iter().enumerate().for_each(|(i, e)| {
-		storage::unhashed::put(well_known_keys::EXTRINSIC_INDEX, &(i as u32));
-		execute_transaction_backend(&e).unwrap_or_else(|_| panic!("Invalid transaction"));
-		storage::unhashed::kill(well_known_keys::EXTRINSIC_INDEX);
-	});
-
-	// check storage root.
-	let storage_root = storage_root().into();
-	info_expect_equal_hash(&storage_root, &header.state_root);
-	assert!(storage_root == header.state_root, "Storage root must match that calculated.");
-
-	// check digest
-	let digest = &mut header.digest;
-	if let Some(storage_changes_root) = storage_changes_root(header.parent_hash.into()) {
-		digest.push(generic::DigestItem::ChangesTrieRoot(storage_changes_root.into()));
-	}
-	if let Some(new_authorities) = <NewAuthorities>::take() {
-		digest.push(generic::DigestItem::AuthoritiesChange(new_authorities));
+		digest.push(generic::DigestItem::Consensus(*b"aura", new_authorities.encode()));
+		digest.push(generic::DigestItem::Consensus(*b"babe", new_authorities.encode()));
 	}
 }
 
@@ -224,7 +212,8 @@ pub fn finalize_block() -> Header {
 		digest.push(generic::DigestItem::ChangesTrieRoot(storage_changes_root));
 	}
 	if let Some(new_authorities) = <NewAuthorities>::take() {
-		digest.push(generic::DigestItem::AuthoritiesChange(new_authorities));
+		digest.push(generic::DigestItem::Consensus(*b"aura", new_authorities.encode()));
+		digest.push(generic::DigestItem::Consensus(*b"babe", new_authorities.encode()));
 	}
 
 	Header {
@@ -309,25 +298,27 @@ fn info_expect_equal_hash(given: &Hash, expected: &Hash) {
 mod tests {
 	use super::*;
 
-	use runtime_io::{with_externalities, twox_128, blake2_256, TestExternalities};
-	use parity_codec::{Joiner, KeyedVec};
+	use runtime_io::{with_externalities, TestExternalities};
 	use substrate_test_runtime_client::{AuthorityKeyring, AccountKeyring};
 	use crate::{Header, Transfer};
 	use primitives::{Blake2Hasher, map};
-	use primitives::storage::well_known_keys;
 	use substrate_executor::WasmExecutor;
 
 	const WASM_CODE: &'static [u8] =
 			include_bytes!("../wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm");
 
 	fn new_test_ext() -> TestExternalities<Blake2Hasher> {
+		let authorities = vec![
+			AuthorityKeyring::Alice.to_raw_public(),
+			AuthorityKeyring::Bob.to_raw_public(),
+			AuthorityKeyring::Charlie.to_raw_public()
+		];
 		TestExternalities::new(map![
 			twox_128(b"latest").to_vec() => vec![69u8; 32],
-			twox_128(well_known_keys::AUTHORITY_COUNT).to_vec() => vec![].and(&3u32),
-			twox_128(&0u32.to_keyed_vec(well_known_keys::AUTHORITY_PREFIX)).to_vec() => AuthorityKeyring::Alice.to_raw_public().to_vec(),
-			twox_128(&1u32.to_keyed_vec(well_known_keys::AUTHORITY_PREFIX)).to_vec() => AuthorityKeyring::Bob.to_raw_public().to_vec(),
-			twox_128(&2u32.to_keyed_vec(well_known_keys::AUTHORITY_PREFIX)).to_vec() => AuthorityKeyring::Charlie.to_raw_public().to_vec(),
-			blake2_256(&AccountKeyring::Alice.to_raw_public().to_keyed_vec(b"balance:")).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0]
+			twox_128(b"sys:auth").to_vec() => authorities.encode(),
+			blake2_256(&AccountKeyring::Alice.to_raw_public().to_keyed_vec(b"balance:")).to_vec() => {
+				vec![111u8, 0, 0, 0, 0, 0, 0, 0]
+			}
 		])
 	}