Refactor key management (#3296)

* Add Call type to extensible transactions.

Cleanup some naming

* Merge Resource and BlockExhausted into just Exhausted

* Fix

* Another fix

* Call

* Some fixes

* Fix srml tests.

* Fix all tests.

* Refactor crypto so each application of it has its own type.

* Introduce new AuthorityProvider API into Aura

This will eventually allow for dynamic determination of authority
keys and avoid having to set them directly on CLI.

* Introduce authority determinator for Babe.

Experiment with modular consensus API.

* Work in progress to introduce KeyTypeId and avoid polluting API
with validator IDs

* Finish up drafting imonline

* Rework offchain workers API.

* Rework API implementation.

* Make it compile for wasm, simplify app_crypto.

* Fix compilation of im-online.

* Fix compilation of im-online.

* Fix more compilation errors.

* Make it compile.

* Fixing tests.

* Rewrite `keystore`

* Fix session tests

* Bring back `TryFrom`'s'

* Fix `srml-grandpa`

* Fix `srml-aura`

* Fix consensus babe

* More fixes

* Make service generate keys from dev_seed

* Build fixes

* Remove offchain tests

* More fixes and cleanups

* Fixes finality grandpa

* Fix `consensus-aura`

* Fix cli

* Fix `node-cli`

* Fix chain_spec builder

* Fix doc tests

* Add authority getter for grandpa.

* Test fix

* Fixes

* Make keystore accessible from the runtime

* Move app crypto to its own crate

* Update `Cargo.lock`

* Make the crypto stuff usable from the runtime

* Adds some runtime crypto tests

* Use last finalized block for grandpa authority

* Fix warning

* Adds `SessionKeys` runtime api

* Remove `FinalityPair` and `ConsensusPair`

* Minor governance tweaks to get it inline with docs.

* Make the governance be up to date with the docs.

* Build fixes.

* Generate the inital session keys

* Failing keystore is a hard error

* Make babe work again

* Fix grandpa

* Fix tests

* Disable `keystore` in consensus critical stuff

* Build fix.

* ImOnline supports multiple authorities at once.

* Update core/application-crypto/src/ed25519.rs

* Merge branch 'master' into gav-in-progress

* Remove unneeded code for now.

* Some `session` testing

* Support querying the public keys

* Cleanup offchain

* Remove warnings

* More cleanup

* Apply suggestions from code review

Co-Authored-By: Benjamin Kampmann <ben.kampmann@googlemail.com>

* More cleanups

* JSONRPC API for setting keys.

Also, rename traits::KeyStore* -> traits::BareCryptoStore*

* Bad merge

* Fix integration tests

* Fix test build

* Test fix

* Fixes

* Warnings

* Another warning

* Bump version.

substrate/srml/session/src/historical.rs

@@ -100,9 +100,9 @@ impl<T: Trait> Module<T> {
 /// Specialization of the crate-level `OnSessionEnding` which returns the old
 /// set of full identification when changing the validator set.
 pub trait OnSessionEnding<ValidatorId, FullIdentification>: crate::OnSessionEnding<ValidatorId> {
-	/// Returns the set of new validators, if any, along with the old validators
-	/// and their full identifications.
-	fn on_session_ending(ending: SessionIndex, applied_at: SessionIndex)
+	/// If there was a validator set change, its returns the set of new validators along with the
+	/// old validators and their full identifications.
+	fn on_session_ending(ending: SessionIndex, will_apply_at: SessionIndex)
 		-> Option<(Vec<ValidatorId>, Vec<(ValidatorId, FullIdentification)>)>;
 }
 
@@ -312,11 +312,8 @@ impl<T: Trait, D: AsRef<[u8]>> srml_support::traits::KeyOwnerProofSystem<(KeyTyp
 mod tests {
 	use super::*;
 	use runtime_io::with_externalities;
-	use primitives::Blake2Hasher;
-	use sr_primitives::{
-		traits::OnInitialize,
-		testing::{UintAuthorityId, UINT_DUMMY_KEY},
-	};
+	use primitives::{Blake2Hasher, crypto::key_types::DUMMY};
+	use sr_primitives::{traits::OnInitialize, testing::UintAuthorityId};
 	use crate::mock::{
 		NEXT_VALIDATORS, force_new_session,
 		set_next_validators, Test, System, Session,
@@ -329,7 +326,7 @@ mod tests {
 		let mut t = system::GenesisConfig::default().build_storage::<Test>().unwrap().0;
 		let (storage, _child_storage) = crate::GenesisConfig::<Test> {
 			keys: NEXT_VALIDATORS.with(|l|
-				l.borrow().iter().cloned().map(|i| (i, UintAuthorityId(i))).collect()
+				l.borrow().iter().cloned().map(|i| (i, UintAuthorityId(i).into())).collect()
 			),
 		}.build_storage().unwrap();
 		t.extend(storage);
@@ -346,15 +343,10 @@ mod tests {
 			Session::on_initialize(1);
 
 			let encoded_key_1 = UintAuthorityId(1).encode();
-			let proof = Historical::prove((UINT_DUMMY_KEY, &encoded_key_1[..])).unwrap();
+			let proof = Historical::prove((DUMMY, &encoded_key_1[..])).unwrap();
 
 			// proof-checking in the same session is OK.
-			assert!(
-				Historical::check_proof(
-					(UINT_DUMMY_KEY, &encoded_key_1[..]),
-					proof.clone(),
-				).is_some()
-			);
+			assert!(Historical::check_proof((DUMMY, &encoded_key_1[..]), proof.clone()).is_some());
 
 			set_next_validators(vec![1, 2, 4]);
 			force_new_session();
@@ -370,12 +362,7 @@ mod tests {
 			assert!(Session::current_index() > proof.session);
 
 			// proof-checking in the next session is also OK.
-			assert!(
-				Historical::check_proof(
-					(UINT_DUMMY_KEY, &encoded_key_1[..]),
-					proof.clone(),
-				).is_some()
-			);
+			assert!(Historical::check_proof((DUMMY, &encoded_key_1[..]), proof.clone()).is_some());
 
 			set_next_validators(vec![1, 2, 5]);
 

substrate/srml/session/src/lib.rs

@@ -121,9 +121,9 @@
 
 use rstd::{prelude::*, marker::PhantomData, ops::{Sub, Rem}};
 use codec::Decode;
-use sr_primitives::KeyTypeId;
+use sr_primitives::{KeyTypeId, AppKey};
 use sr_primitives::weights::SimpleDispatchInfo;
-use sr_primitives::traits::{Convert, Zero, Member, OpaqueKeys, TypedKey};
+use sr_primitives::traits::{Convert, Zero, Member, OpaqueKeys};
 use srml_support::{
 	dispatch::Result, ConsensusEngineId, StorageValue, StorageDoubleMap, for_each_tuple,
 	decl_module, decl_event, decl_storage,
@@ -172,10 +172,13 @@ pub trait OnSessionEnding<ValidatorId> {
 	/// Handle the fact that the session is ending, and optionally provide the new validator set.
 	///
 	/// `ending_index` is the index of the currently ending session.
-	/// The returned validator set, if any, will not be applied until `next_index`.
-	/// `next_index` is guaranteed to be at least `ending_index + 1`, since session indices don't
-	/// repeat.
-	fn on_session_ending(ending_index: SessionIndex, next_index: SessionIndex) -> Option<Vec<ValidatorId>>;
+	/// The returned validator set, if any, will not be applied until `will_apply_at`.
+	/// `will_apply_at` is guaranteed to be at least `ending_index + 1`, since session indices don't
+	/// repeat, but it could be some time after in case we are staging authority set changes.
+	fn on_session_ending(
+		ending_index: SessionIndex,
+		will_apply_at: SessionIndex
+	) -> Option<Vec<ValidatorId>>;
 }
 
 impl<A> OnSessionEnding<A> for () {
@@ -198,7 +201,7 @@ pub trait SessionHandler<ValidatorId> {
 /// One session-key type handler.
 pub trait OneSessionHandler<ValidatorId> {
 	/// The key type expected.
-	type Key: Decode + Default + TypedKey;
+	type Key: Decode + Default + AppKey;
 
 	fn on_new_session<'a, I: 'a>(changed: bool, validators: I, queued_validators: I)
 		where I: Iterator<Item=(&'a ValidatorId, Self::Key)>, ValidatorId: 'a;
@@ -222,10 +225,10 @@ macro_rules! impl_session_handlers {
 			) {
 				$(
 					let our_keys: Box<dyn Iterator<Item=_>> = Box::new(validators.iter()
-						.map(|k| (&k.0, k.1.get::<$t::Key>(<$t::Key as TypedKey>::KEY_TYPE)
+						.map(|k| (&k.0, k.1.get::<$t::Key>(<$t::Key as AppKey>::ID)
 							.unwrap_or_default())));
 					let queued_keys: Box<dyn Iterator<Item=_>> = Box::new(queued_validators.iter()
-						.map(|k| (&k.0, k.1.get::<$t::Key>(<$t::Key as TypedKey>::KEY_TYPE)
+						.map(|k| (&k.0, k.1.get::<$t::Key>(<$t::Key as AppKey>::ID)
 							.unwrap_or_default())));
 					$t::on_new_session(changed, our_keys, queued_keys);
 				)*
@@ -562,7 +565,7 @@ mod tests {
 	use super::*;
 	use srml_support::assert_ok;
 	use runtime_io::with_externalities;
-	use primitives::Blake2Hasher;
+	use primitives::{Blake2Hasher, crypto::key_types::DUMMY};
 	use sr_primitives::{
 		traits::OnInitialize,
 		testing::UintAuthorityId,
@@ -576,7 +579,7 @@ mod tests {
 		let mut t = system::GenesisConfig::default().build_storage::<Test>().unwrap();
 		GenesisConfig::<Test> {
 			keys: NEXT_VALIDATORS.with(|l|
-				l.borrow().iter().cloned().map(|i| (i, UintAuthorityId(i))).collect()
+				l.borrow().iter().cloned().map(|i| (i, UintAuthorityId(i).into())).collect()
 			),
 		}.assimilate_storage(&mut t.0, &mut t.1).unwrap();
 		runtime_io::TestExternalities::new_with_children(t)
@@ -599,8 +602,8 @@ mod tests {
 	#[test]
 	fn put_get_keys() {
 		with_externalities(&mut new_test_ext(), || {
-			Session::put_keys(&10, &UintAuthorityId(10));
-			assert_eq!(Session::load_keys(&10), Some(UintAuthorityId(10)));
+			Session::put_keys(&10, &UintAuthorityId(10).into());
+			assert_eq!(Session::load_keys(&10), Some(UintAuthorityId(10).into()));
 		})
 	}
 
@@ -609,9 +612,9 @@ mod tests {
 		let mut ext = new_test_ext();
 		with_externalities(&mut ext, || {
 			assert_eq!(Session::validators(), vec![1, 2, 3]);
-			assert_eq!(Session::load_keys(&1), Some(UintAuthorityId(1)));
+			assert_eq!(Session::load_keys(&1), Some(UintAuthorityId(1).into()));
 
-			let id = <UintAuthorityId as TypedKey>::KEY_TYPE;
+			let id = DUMMY;
 			assert_eq!(Session::key_owner(id, UintAuthorityId(1).get_raw(id)), Some(1));
 
 			Session::on_free_balance_zero(&1);
@@ -629,8 +632,8 @@ mod tests {
 			force_new_session();
 			initialize_block(1);
 			assert_eq!(Session::queued_keys(), vec![
-				(1, UintAuthorityId(1)),
-				(2, UintAuthorityId(2)),
+				(1, UintAuthorityId(1).into()),
+				(2, UintAuthorityId(2).into()),
 			]);
 			assert_eq!(Session::validators(), vec![1, 2, 3]);
 			assert_eq!(authorities(), vec![UintAuthorityId(1), UintAuthorityId(2), UintAuthorityId(3)]);
@@ -638,20 +641,20 @@ mod tests {
 			force_new_session();
 			initialize_block(2);
 			assert_eq!(Session::queued_keys(), vec![
-				(1, UintAuthorityId(1)),
-				(2, UintAuthorityId(2)),
+				(1, UintAuthorityId(1).into()),
+				(2, UintAuthorityId(2).into()),
 			]);
 			assert_eq!(Session::validators(), vec![1, 2]);
 			assert_eq!(authorities(), vec![UintAuthorityId(1), UintAuthorityId(2)]);
 
 			set_next_validators(vec![1, 2, 4]);
-			assert_ok!(Session::set_keys(Origin::signed(4), UintAuthorityId(4), vec![]));
+			assert_ok!(Session::set_keys(Origin::signed(4), UintAuthorityId(4).into(), vec![]));
 			force_new_session();
 			initialize_block(3);
 			assert_eq!(Session::queued_keys(), vec![
-				(1, UintAuthorityId(1)),
-				(2, UintAuthorityId(2)),
-				(4, UintAuthorityId(4)),
+				(1, UintAuthorityId(1).into()),
+				(2, UintAuthorityId(2).into()),
+				(4, UintAuthorityId(4).into()),
 			]);
 			assert_eq!(Session::validators(), vec![1, 2]);
 			assert_eq!(authorities(), vec![UintAuthorityId(1), UintAuthorityId(2)]);
@@ -659,9 +662,9 @@ mod tests {
 			force_new_session();
 			initialize_block(4);
 			assert_eq!(Session::queued_keys(), vec![
-				(1, UintAuthorityId(1)),
-				(2, UintAuthorityId(2)),
-				(4, UintAuthorityId(4)),
+				(1, UintAuthorityId(1).into()),
+				(2, UintAuthorityId(2).into()),
+				(4, UintAuthorityId(4).into()),
 			]);
 			assert_eq!(Session::validators(), vec![1, 2, 4]);
 			assert_eq!(authorities(), vec![UintAuthorityId(1), UintAuthorityId(2), UintAuthorityId(4)]);
@@ -704,7 +707,7 @@ mod tests {
 
 			// Block 3: Set new key for validator 2; no visible change.
 			initialize_block(3);
-			assert_ok!(Session::set_keys(Origin::signed(2), UintAuthorityId(5), vec![]));
+			assert_ok!(Session::set_keys(Origin::signed(2), UintAuthorityId(5).into(), vec![]));
 			assert_eq!(authorities(), vec![UintAuthorityId(1), UintAuthorityId(2), UintAuthorityId(3)]);
 
 			// Block 4: Session rollover; no visible change.
@@ -726,11 +729,11 @@ mod tests {
 		with_externalities(&mut new_test_ext(), || {
 			System::set_block_number(1);
 			Session::on_initialize(1);
-			assert!(Session::set_keys(Origin::signed(4), UintAuthorityId(1), vec![]).is_err());
-			assert!(Session::set_keys(Origin::signed(1), UintAuthorityId(10), vec![]).is_ok());
+			assert!(Session::set_keys(Origin::signed(4), UintAuthorityId(1).into(), vec![]).is_err());
+			assert!(Session::set_keys(Origin::signed(1), UintAuthorityId(10).into(), vec![]).is_ok());
 
 			// is fine now that 1 has migrated off.
-			assert!(Session::set_keys(Origin::signed(4), UintAuthorityId(1), vec![]).is_ok());
+			assert!(Session::set_keys(Origin::signed(4), UintAuthorityId(1).into(), vec![]).is_ok());
 		});
 	}
 
@@ -760,7 +763,7 @@ mod tests {
 			initialize_block(5);
 			assert!(!session_changed());
 
-			assert_ok!(Session::set_keys(Origin::signed(2), UintAuthorityId(5), vec![]));
+			assert_ok!(Session::set_keys(Origin::signed(2), UintAuthorityId(5).into(), vec![]));
 			force_new_session();
 			initialize_block(6);
 			assert!(!session_changed());
@@ -799,4 +802,18 @@ mod tests {
 
 		assert!(P::should_end_session(13));
 	}
+
+	#[test]
+	fn session_keys_generate_output_works_as_set_keys_input() {
+		with_externalities(&mut new_test_ext(), || {
+			let new_keys = mock::MockSessionKeys::generate(None);
+			assert_ok!(
+				Session::set_keys(
+					Origin::signed(2),
+					<mock::Test as Trait>::Keys::decode(&mut &new_keys[..]).expect("Decode keys"),
+					vec![],
+				)
+			);
+		});
+	}
 }

substrate/srml/session/src/mock.rs

@@ -19,13 +19,24 @@
 use super::*;
 use std::cell::RefCell;
 use srml_support::{impl_outer_origin, parameter_types};
-use primitives::H256;
+use primitives::{crypto::key_types::DUMMY, H256};
 use sr_primitives::{
-	Perbill,
-	traits::{BlakeTwo256, IdentityLookup, ConvertInto},
+	Perbill, impl_opaque_keys, traits::{BlakeTwo256, IdentityLookup, ConvertInto},
 	testing::{Header, UintAuthorityId}
 };
 
+impl_opaque_keys! {
+	pub struct MockSessionKeys {
+		#[id(DUMMY)]
+		pub dummy: UintAuthorityId,
+	}
+}
+
+impl From<UintAuthorityId> for MockSessionKeys {
+	fn from(dummy: UintAuthorityId) -> Self {
+		Self { dummy }
+	}
+}
 
 impl_outer_origin! {
 	pub enum Origin for Test {}
@@ -58,7 +69,9 @@ impl SessionHandler<u64> for TestSessionHandler {
 	) {
 		SESSION_CHANGED.with(|l| *l.borrow_mut() = changed);
 		AUTHORITIES.with(|l|
-			*l.borrow_mut() = validators.iter().map(|(_, id)| id.get::<UintAuthorityId>(0).unwrap_or_default()).collect()
+			*l.borrow_mut() = validators.iter()
+				.map(|(_, id)| id.get::<UintAuthorityId>(DUMMY).unwrap_or_default())
+				.collect()
 		);
 	}
 	fn on_disabled(_validator_index: usize) {}
@@ -119,10 +132,12 @@ parameter_types! {
 	pub const MinimumPeriod: u64 = 5;
 	pub const AvailableBlockRatio: Perbill = Perbill::one();
 }
+
 impl system::Trait for Test {
 	type Origin = Origin;
 	type Index = u64;
 	type BlockNumber = u64;
+	type Call = ();
 	type Hash = H256;
 	type Hashing = BlakeTwo256;
 	type AccountId = u64;
@@ -135,13 +150,13 @@ impl system::Trait for Test {
 	type AvailableBlockRatio = AvailableBlockRatio;
 	type MaximumBlockLength = MaximumBlockLength;
 }
+
 impl timestamp::Trait for Test {
 	type Moment = u64;
 	type OnTimestampSet = ();
 	type MinimumPeriod = MinimumPeriod;
 }
 
-
 impl Trait for Test {
 	type ShouldEndSession = TestShouldEndSession;
 	#[cfg(feature = "historical")]
@@ -151,7 +166,7 @@ impl Trait for Test {
 	type SessionHandler = TestSessionHandler;
 	type ValidatorId = u64;
 	type ValidatorIdOf = ConvertInto;
-	type Keys = UintAuthorityId;
+	type Keys = MockSessionKeys;
 	type Event = ();
 	type SelectInitialValidators = ();
 }