New sessions, kill consensus module (#2802)

* Draft of new sessions

* Reintroduce tuple impls

* Move staking module to new session API

* More work on staking and grandpa.

* Use iterator to avoid cloning and tuple macro

* Make runtime build again

* Polish the OpaqueKeys devex

* Move consensus logic into system & aura.

* Fix up system module

* Get build mostly going. Stuck at service.rs

* Building again

* Update srml/staking/src/lib.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Refactoring out Consensus module, AuthorityIdOf, &c.

* Refactored out DigestItem::AuthoritiesChanged. Building.

* Remove tentative code

* Remove invalid comment

* Make Seal opaque and introduce nice methods for handling opaque items.

* Start to use proper digest for Aura authorities tracking.

* Fix up grandpa, remove system::Raw/Log

* Refactor Grandpa to use new logging infrastructure.

Also make authorityid/sessionkey static. Switch over to storing
authorities in a straight Vec.

* Building again

* Tidy up some AuthorityIds

* Expunge most of the rest of the AuthorityKey confusion.

Also, de-generify Babe and re-generify Aura.

* Remove cruft

* Untangle last of the `AuthorityId`s.

* Sort out finality_tracker

* Refactor median getting

* Apply suggestions from code review

Co-Authored-By: Robert Habermeier <rphmeier@gmail.com>

* Session tests works

* Update core/sr-primitives/src/generic/digest.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Session tests works

* Fix for staking from @dvc94ch

* log an error

* fix test runtime build

* Some test fixes

* Staking mock update to new session api.

* Fix build.

* Move OpaqueKeys to primitives.

* Use on_initialize instead of check_rotate_session.

* Update tests to new staking api.

* fixup mock

* Fix bond_extra_and_withdraw_unbonded_works.

* Fix bond_with_little_staked_value_bounded_by_slot_stake.

* Fix bond_with_no_staked_value.

* Fix change_controller_works.

* Fix less_than_needed_candidates_works.

* Fix multi_era_reward_should_work.

* Fix nominating_and_rewards_should_work.

* Fix nominators_also_get_slashed.

* Fix phragmen_large_scale_test.

* Fix phragmen_poc_works.

* Fix phragmen_score_should_be_accurate_on_large_stakes.

* Fix phragmen_should_not_overflow.

* Fix reward_destination_works.

* Fix rewards_should_work.

* Fix sessions_and_eras_should_work.

* Fix slot_stake_is_least_staked_validator.

* Fix too_many_unbond_calls_should_not_work.

* Fix wrong_vote_is_null.

* Fix runtime.

* Fix wasm runtime build.

* Update Cargo.lock

* Fix warnings.

* Fix grandpa tests.

* Fix test-runtime build.

* Fix template node build.

* Fix stuff.

* Update Cargo.lock to fix CI

* Re-add missing AuRa logs

Runtimes are required to know about every digest they receive ― they
panic otherwise.  This re-adds support for AuRa pre-runtime digests.

* Update core/consensus/babe/src/digest.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* Kill log trait and all that jazz.

* Refactor staking tests.

* Fix ci runtime wasm check.

* Line length 120.

* Make tests build again

* Remove trailing commas in function declarations

The `extern_functions!` macro doesn’t like them, perhaps due to a bug in
rustc.

* Fix type error

* Fix compilation errors

* Fix a test

* Another couple of fixes

* Fix another test

* More test fixes

* Another test fix

* Bump runtime.

* Wrap long line

* Fix build, remove redundant code.

* Issue to track TODO

* Leave the benchmark code alone.

* Fix missing `std::time::{Instant, Duration}`

* Indentation

* Aura ConsensusLog as enum

substrate/core/test-runtime/src/genesismap.rs

@@ -18,11 +18,10 @@
 
 use std::collections::HashMap;
 use runtime_io::{blake2_256, twox_128};
-use super::AccountId;
+use super::{AuthorityId, AccountId};
 use parity_codec::{Encode, KeyedVec, Joiner};
 use primitives::{ChangesTrieConfiguration, map, storage::well_known_keys};
 use runtime_primitives::traits::Block;
-use primitives::sr25519::Public as AuthorityId;
 
 /// Configuration of a general Substrate test genesis block.
 pub struct GenesisConfig {
@@ -58,16 +57,12 @@ impl GenesisConfig {
 				#[cfg(feature = "include-wasm-blob")]
 				(well_known_keys::CODE.into(), wasm_runtime),
 				(well_known_keys::HEAP_PAGES.into(), vec![].and(&(16 as u64))),
-				(well_known_keys::AUTHORITY_COUNT.into(), vec![].and(&(self.authorities.len() as u32))),
 			].into_iter())
-			.chain(self.authorities.iter()
-				.enumerate()
-				.map(|(i, account)| ((i as u32).to_keyed_vec(well_known_keys::AUTHORITY_PREFIX), vec![].and(account)))
-			)
 			.collect();
 		if let Some(ref changes_trie_config) = self.changes_trie_config {
 			map.insert(well_known_keys::CHANGES_TRIE_CONFIG.to_vec(), changes_trie_config.encode());
 		}
+		map.insert(twox_128(&b"sys:auth"[..])[..].to_vec(), self.authorities.encode());
 		map
 	}
 }

substrate/core/test-runtime/src/lib.rs

@@ -39,7 +39,7 @@ use runtime_primitives::{
 	transaction_validity::TransactionValidity,
 	traits::{
 		BlindCheckable, BlakeTwo256, Block as BlockT, Extrinsic as ExtrinsicT,
-		GetNodeBlockType, GetRuntimeBlockType, AuthorityIdFor, Verify,
+		GetNodeBlockType, GetRuntimeBlockType, Verify
 	},
 };
 use runtime_version::RuntimeVersion;
@@ -49,6 +49,12 @@ use primitives::{sr25519, OpaqueMetadata};
 use runtime_version::NativeVersion;
 use inherents::{CheckInherentsResult, InherentData};
 use cfg_if::cfg_if;
+pub use consensus_babe::AuthorityId;
+
+// Ensure Babe and Aura use the same crypto to simplify things a bit.
+pub type AuraId = AuthorityId;
+// Ensure Babe and Aura use the same crypto to simplify things a bit.
+pub type BabeId = AuthorityId;
 
 /// Test runtime version.
 pub const VERSION: RuntimeVersion = RuntimeVersion {
@@ -146,10 +152,6 @@ impl Extrinsic {
 	}
 }
 
-/// The signature type used by authorities.
-pub type AuthoritySignature = sr25519::Signature;
-/// The identity type used by authorities.
-pub type AuthorityId = <AuthoritySignature as Verify>::Signer;
 /// The signature type used by accounts/transactions.
 pub type AccountSignature = sr25519::Signature;
 /// An identifier for an account on this system.
@@ -161,13 +163,13 @@ pub type BlockNumber = u64;
 /// Index of a transaction.
 pub type Index = u64;
 /// The item of a block digest.
-pub type DigestItem = runtime_primitives::generic::DigestItem<H256, AuthorityId, AuthoritySignature>;
+pub type DigestItem = runtime_primitives::generic::DigestItem<H256>;
 /// The digest of a block.
-pub type Digest = runtime_primitives::generic::Digest<DigestItem>;
+pub type Digest = runtime_primitives::generic::Digest<H256>;
 /// A test block.
 pub type Block = runtime_primitives::generic::Block<Header, Extrinsic>;
 /// A test block's header.
-pub type Header = runtime_primitives::generic::Header<BlockNumber, BlakeTwo256, DigestItem>;
+pub type Header = runtime_primitives::generic::Header<BlockNumber, BlakeTwo256>;
 
 /// Run whatever tests we have.
 pub fn run_tests(mut input: &[u8]) -> Vec<u8> {
@@ -455,8 +457,9 @@ cfg_if! {
 				}
 			}
 
-			impl consensus_aura::AuraApi<Block> for Runtime {
+			impl consensus_aura::AuraApi<Block, AuraId> for Runtime {
 				fn slot_duration() -> u64 { 1 }
+				fn authorities() -> Vec<AuraId> { system::authorities() }
 			}
 
 			impl consensus_babe::BabeApi<Block> for Runtime {
@@ -467,6 +470,7 @@ cfg_if! {
 						threshold: std::u64::MAX,
 					}
 				}
+				fn authorities() -> Vec<BabeId> { system::authorities() }
 			}
 
 			impl offchain_primitives::OffchainWorkerApi<Block> for Runtime {
@@ -475,12 +479,6 @@ cfg_if! {
 					runtime_io::submit_transaction(&ex).unwrap();
 				}
 			}
-
-			impl consensus_authorities::AuthoritiesApi<Block> for Runtime {
-				fn authorities() -> Vec<AuthorityIdFor<Block>> {
-					system::authorities()
-				}
-			}
 		}
 	} else {
 		impl_runtime_apis! {
@@ -603,8 +601,9 @@ cfg_if! {
 				}
 			}
 
-			impl consensus_aura::AuraApi<Block> for Runtime {
+			impl consensus_aura::AuraApi<Block, AuraId> for Runtime {
 				fn slot_duration() -> u64 { 1 }
+				fn authorities() -> Vec<AuraId> { system::authorities() }
 			}
 
 			impl consensus_babe::BabeApi<Block> for Runtime {
@@ -615,6 +614,7 @@ cfg_if! {
 						threshold: core::u64::MAX,
 					}
 				}
+				fn authorities() -> Vec<BabeId> { system::authorities() }
 			}
 
 			impl offchain_primitives::OffchainWorkerApi<Block> for Runtime {
@@ -623,12 +623,6 @@ cfg_if! {
 					runtime_io::submit_transaction(&ex).unwrap()
 				}
 			}
-
-			impl consensus_authorities::AuthoritiesApi<Block> for Runtime {
-				fn authorities() -> Vec<AuthorityIdFor<Block>> {
-					system::authorities()
-				}
-			}
 		}
 	}
 }

substrate/core/test-runtime/src/system.rs

@@ -21,13 +21,14 @@ use rstd::prelude::*;
 use runtime_io::{storage_root, enumerated_trie_root, storage_changes_root, twox_128, blake2_256};
 use runtime_support::storage::{self, StorageValue, StorageMap};
 use runtime_support::storage_items;
-use runtime_primitives::traits::{Hash as HashT, BlakeTwo256, Digest as DigestT, Header as _};
+use runtime_primitives::traits::{Hash as HashT, BlakeTwo256, Header as _};
 use runtime_primitives::generic;
 use runtime_primitives::{ApplyError, ApplyOutcome, ApplyResult, transaction_validity::TransactionValidity};
 use parity_codec::{KeyedVec, Encode};
-use super::{AccountId, BlockNumber, Extrinsic, Transfer, H256 as Hash, Block, Header, Digest};
+use super::{
+	AccountId, BlockNumber, Extrinsic, Transfer, H256 as Hash, Block, Header, Digest, AuthorityId
+};
 use primitives::{Blake2Hasher, storage::well_known_keys};
-use primitives::sr25519::Public as AuthorityId;
 
 const NONCE_OF: &[u8] = b"nonce:";
 const BALANCE_OF: &[u8] = b"balance:";
@@ -39,6 +40,7 @@ storage_items! {
 	ParentHash: b"sys:pha" => required Hash;
 	NewAuthorities: b"sys:new_auth" => Vec<AuthorityId>;
 	StorageDigest: b"sys:digest" => Digest;
+	Authorities get(authorities): b"sys:auth" => default Vec<AuthorityId>;
 }
 
 pub fn balance_of_key(who: AccountId) -> Vec<u8> {
@@ -53,17 +55,6 @@ pub fn nonce_of(who: AccountId) -> u64 {
 	storage::hashed::get_or(&blake2_256, &who.to_keyed_vec(NONCE_OF), 0)
 }
 
-/// Get authorities at given block.
-pub fn authorities() -> Vec<AuthorityId> {
-	let len: u32 = storage::unhashed::get(well_known_keys::AUTHORITY_COUNT)
-		.expect("There are always authorities in test-runtime");
-	(0..len)
-		.map(|i| storage::unhashed::get(&i.to_keyed_vec(well_known_keys::AUTHORITY_PREFIX))
-			.expect("Authority is properly encoded in test-runtime")
-		)
-		.collect()
-}
-
 pub fn initialize_block(header: &Header) {
 	// populate environment.
 	<Number>::put(&header.number);
@@ -80,8 +71,25 @@ pub fn take_block_number() -> Option<BlockNumber> {
 	Number::take()
 }
 
+#[derive(Copy, Clone)]
+enum Mode {
+	Verify,
+	Overwrite,
+}
+
 /// Actually execute all transitioning for `block`.
 pub fn polish_block(block: &mut Block) {
+	execute_block_with_state_root_handler(block, Mode::Overwrite);
+}
+
+pub fn execute_block(mut block: Block) {
+	execute_block_with_state_root_handler(&mut block, Mode::Verify);
+}
+
+fn execute_block_with_state_root_handler(
+	block: &mut Block,
+	mode: Mode,
+) {
 	let header = &mut block.header;
 
 	// check transaction trie root represents the transactions.
@@ -89,7 +97,11 @@ pub fn polish_block(block: &mut Block) {
 	let txs = txs.iter().map(Vec::as_slice).collect::<Vec<_>>();
 	let txs_root = enumerated_trie_root::<Blake2Hasher>(&txs).into();
 	info_expect_equal_hash(&txs_root, &header.extrinsics_root);
-	header.extrinsics_root = txs_root;
+	if let Mode::Overwrite = mode {
+		header.extrinsics_root = txs_root;
+	} else {
+		assert!(txs_root == header.extrinsics_root, "Transaction trie root must be valid.");
+	}
 
 	// execute transactions
 	block.extrinsics.iter().enumerate().for_each(|(i, e)| {
@@ -98,7 +110,14 @@ pub fn polish_block(block: &mut Block) {
 		storage::unhashed::kill(well_known_keys::EXTRINSIC_INDEX);
 	});
 
-	header.state_root = storage_root().into();
+	if let Mode::Overwrite = mode {
+		header.state_root = storage_root().into();
+	} else {
+		// check storage root.
+		let storage_root = storage_root().into();
+		info_expect_equal_hash(&storage_root, &header.state_root);
+		assert!(storage_root == header.state_root, "Storage root must match that calculated.");
+	}
 
 	// check digest
 	let digest = &mut header.digest;
@@ -106,39 +125,8 @@ pub fn polish_block(block: &mut Block) {
 		digest.push(generic::DigestItem::ChangesTrieRoot(storage_changes_root.into()));
 	}
 	if let Some(new_authorities) = <NewAuthorities>::take() {
-		digest.push(generic::DigestItem::AuthoritiesChange(new_authorities));
-	}
-}
-
-pub fn execute_block(mut block: Block) {
-	let header = &mut block.header;
-
-	// check transaction trie root represents the transactions.
-	let txs = block.extrinsics.iter().map(Encode::encode).collect::<Vec<_>>();
-	let txs = txs.iter().map(Vec::as_slice).collect::<Vec<_>>();
-	let txs_root = enumerated_trie_root::<Blake2Hasher>(&txs).into();
-	info_expect_equal_hash(&txs_root, &header.extrinsics_root);
-	assert!(txs_root == header.extrinsics_root, "Transaction trie root must be valid.");
-
-	// execute transactions
-	block.extrinsics.into_iter().enumerate().for_each(|(i, e)| {
-		storage::unhashed::put(well_known_keys::EXTRINSIC_INDEX, &(i as u32));
-		execute_transaction_backend(&e).unwrap_or_else(|_| panic!("Invalid transaction"));
-		storage::unhashed::kill(well_known_keys::EXTRINSIC_INDEX);
-	});
-
-	// check storage root.
-	let storage_root = storage_root().into();
-	info_expect_equal_hash(&storage_root, &header.state_root);
-	assert!(storage_root == header.state_root, "Storage root must match that calculated.");
-
-	// check digest
-	let digest = &mut header.digest;
-	if let Some(storage_changes_root) = storage_changes_root(header.parent_hash.into()) {
-		digest.push(generic::DigestItem::ChangesTrieRoot(storage_changes_root.into()));
-	}
-	if let Some(new_authorities) = <NewAuthorities>::take() {
-		digest.push(generic::DigestItem::AuthoritiesChange(new_authorities));
+		digest.push(generic::DigestItem::Consensus(*b"aura", new_authorities.encode()));
+		digest.push(generic::DigestItem::Consensus(*b"babe", new_authorities.encode()));
 	}
 }
 
@@ -224,7 +212,8 @@ pub fn finalize_block() -> Header {
 		digest.push(generic::DigestItem::ChangesTrieRoot(storage_changes_root));
 	}
 	if let Some(new_authorities) = <NewAuthorities>::take() {
-		digest.push(generic::DigestItem::AuthoritiesChange(new_authorities));
+		digest.push(generic::DigestItem::Consensus(*b"aura", new_authorities.encode()));
+		digest.push(generic::DigestItem::Consensus(*b"babe", new_authorities.encode()));
 	}
 
 	Header {
@@ -309,25 +298,27 @@ fn info_expect_equal_hash(given: &Hash, expected: &Hash) {
 mod tests {
 	use super::*;
 
-	use runtime_io::{with_externalities, twox_128, blake2_256, TestExternalities};
-	use parity_codec::{Joiner, KeyedVec};
+	use runtime_io::{with_externalities, TestExternalities};
 	use substrate_test_runtime_client::{AuthorityKeyring, AccountKeyring};
 	use crate::{Header, Transfer};
 	use primitives::{Blake2Hasher, map};
-	use primitives::storage::well_known_keys;
 	use substrate_executor::WasmExecutor;
 
 	const WASM_CODE: &'static [u8] =
 			include_bytes!("../wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm");
 
 	fn new_test_ext() -> TestExternalities<Blake2Hasher> {
+		let authorities = vec![
+			AuthorityKeyring::Alice.to_raw_public(),
+			AuthorityKeyring::Bob.to_raw_public(),
+			AuthorityKeyring::Charlie.to_raw_public()
+		];
 		TestExternalities::new(map![
 			twox_128(b"latest").to_vec() => vec![69u8; 32],
-			twox_128(well_known_keys::AUTHORITY_COUNT).to_vec() => vec![].and(&3u32),
-			twox_128(&0u32.to_keyed_vec(well_known_keys::AUTHORITY_PREFIX)).to_vec() => AuthorityKeyring::Alice.to_raw_public().to_vec(),
-			twox_128(&1u32.to_keyed_vec(well_known_keys::AUTHORITY_PREFIX)).to_vec() => AuthorityKeyring::Bob.to_raw_public().to_vec(),
-			twox_128(&2u32.to_keyed_vec(well_known_keys::AUTHORITY_PREFIX)).to_vec() => AuthorityKeyring::Charlie.to_raw_public().to_vec(),
-			blake2_256(&AccountKeyring::Alice.to_raw_public().to_keyed_vec(b"balance:")).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0]
+			twox_128(b"sys:auth").to_vec() => authorities.encode(),
+			blake2_256(&AccountKeyring::Alice.to_raw_public().to_keyed_vec(b"balance:")).to_vec() => {
+				vec![111u8, 0, 0, 0, 0, 0, 0, 0]
+			}
 		])
 	}