Refactor key management (#3296)

* Add Call type to extensible transactions.

Cleanup some naming

* Merge Resource and BlockExhausted into just Exhausted

* Fix

* Another fix

* Call

* Some fixes

* Fix srml tests.

* Fix all tests.

* Refactor crypto so each application of it has its own type.

* Introduce new AuthorityProvider API into Aura

This will eventually allow for dynamic determination of authority
keys and avoid having to set them directly on CLI.

* Introduce authority determinator for Babe.

Experiment with modular consensus API.

* Work in progress to introduce KeyTypeId and avoid polluting API
with validator IDs

* Finish up drafting imonline

* Rework offchain workers API.

* Rework API implementation.

* Make it compile for wasm, simplify app_crypto.

* Fix compilation of im-online.

* Fix compilation of im-online.

* Fix more compilation errors.

* Make it compile.

* Fixing tests.

* Rewrite `keystore`

* Fix session tests

* Bring back `TryFrom`'s'

* Fix `srml-grandpa`

* Fix `srml-aura`

* Fix consensus babe

* More fixes

* Make service generate keys from dev_seed

* Build fixes

* Remove offchain tests

* More fixes and cleanups

* Fixes finality grandpa

* Fix `consensus-aura`

* Fix cli

* Fix `node-cli`

* Fix chain_spec builder

* Fix doc tests

* Add authority getter for grandpa.

* Test fix

* Fixes

* Make keystore accessible from the runtime

* Move app crypto to its own crate

* Update `Cargo.lock`

* Make the crypto stuff usable from the runtime

* Adds some runtime crypto tests

* Use last finalized block for grandpa authority

* Fix warning

* Adds `SessionKeys` runtime api

* Remove `FinalityPair` and `ConsensusPair`

* Minor governance tweaks to get it inline with docs.

* Make the governance be up to date with the docs.

* Build fixes.

* Generate the inital session keys

* Failing keystore is a hard error

* Make babe work again

* Fix grandpa

* Fix tests

* Disable `keystore` in consensus critical stuff

* Build fix.

* ImOnline supports multiple authorities at once.

* Update core/application-crypto/src/ed25519.rs

* Merge branch 'master' into gav-in-progress

* Remove unneeded code for now.

* Some `session` testing

* Support querying the public keys

* Cleanup offchain

* Remove warnings

* More cleanup

* Apply suggestions from code review

Co-Authored-By: Benjamin Kampmann <ben.kampmann@googlemail.com>

* More cleanups

* JSONRPC API for setting keys.

Also, rename traits::KeyStore* -> traits::BareCryptoStore*

* Bad merge

* Fix integration tests

* Fix test build

* Test fix

* Fixes

* Warnings

* Another warning

* Bump version.

substrate/core/sr-primitives/src/generic/checked_extrinsic.rs

@@ -45,11 +45,10 @@ for
 where
 	AccountId: Member + MaybeDisplay,
 	Call: Member + Dispatchable<Origin=Origin>,
-	Extra: SignedExtension<AccountId=AccountId>,
+	Extra: SignedExtension<AccountId=AccountId, Call=Call>,
 	Origin: From<Option<AccountId>>,
 {
 	type AccountId = AccountId;
-
 	type Call = Call;
 
 	fn sender(&self) -> Option<&Self::AccountId> {
@@ -61,9 +60,9 @@ where
 		len: usize,
 	) -> TransactionValidity {
 		if let Some((ref id, ref extra)) = self.signed {
-			Extra::validate(extra, id, info, len).into()
+			Extra::validate(extra, id, &self.function, info, len).into()
 		} else {
-			match Extra::validate_unsigned(info, len) {
+			match Extra::validate_unsigned(&self.function, info, len) {
 				Ok(extra) => match U::validate_unsigned(&self.function) {
 					TransactionValidity::Valid(v) =>
 						TransactionValidity::Valid(v.combine_with(extra)),
@@ -79,10 +78,10 @@ where
 		len: usize,
 	) -> Result<DispatchResult, DispatchError> {
 		let maybe_who = if let Some((id, extra)) = self.signed {
-			Extra::pre_dispatch(extra, &id, info, len)?;
+			Extra::pre_dispatch(extra, &id, &self.function, info, len)?;
 			Some(id)
 		} else {
-			Extra::pre_dispatch_unsigned(info, len)?;
+			Extra::pre_dispatch_unsigned(&self.function, info, len)?;
 			None
 		};
 		Ok(self.function.dispatch(Origin::from(maybe_who)))

substrate/core/sr-primitives/src/generic/unchecked_extrinsic.rs

@@ -234,6 +234,7 @@ mod tests {
 	struct TestExtra;
 	impl SignedExtension for TestExtra {
 		type AccountId = u64;
+		type Call = ();
 		type AdditionalSigned = ();
 		fn additional_signed(&self) -> rstd::result::Result<(), &'static str> { Ok(()) }
 	}

substrate/core/sr-primitives/src/lib.rs

@@ -31,10 +31,13 @@ pub use rstd;
 #[doc(hidden)]
 pub use paste;
 
+#[doc(hidden)]
+pub use app_crypto;
+
 #[cfg(feature = "std")]
 pub use runtime_io::{StorageOverlay, ChildrenStorageOverlay};
 
-use rstd::{prelude::*, ops, convert::TryInto};
+use rstd::{prelude::*, ops, convert::{TryInto, TryFrom}};
 use primitives::{crypto, ed25519, sr25519, hash::{H256, H512}};
 use codec::{Encode, Decode};
 
@@ -52,7 +55,8 @@ pub mod transaction_validity;
 pub use generic::{DigestItem, Digest};
 
 /// Re-export this since it's part of the API of this crate.
-pub use primitives::crypto::{key_types, KeyTypeId};
+pub use primitives::crypto::{key_types, KeyTypeId, CryptoType};
+pub use app_crypto::AppKey;
 
 /// A message indicating an invalid signature in extrinsic.
 pub const BAD_SIGNATURE: &str = "bad signature in extrinsic";
@@ -660,8 +664,13 @@ pub struct AnySignature(H512);
 impl Verify for AnySignature {
 	type Signer = sr25519::Public;
 	fn verify<L: Lazy<[u8]>>(&self, mut msg: L, signer: &sr25519::Public) -> bool {
-		runtime_io::sr25519_verify(self.0.as_fixed_bytes(), msg.get(), &signer.0) ||
-			runtime_io::ed25519_verify(self.0.as_fixed_bytes(), msg.get(), &signer.0)
+		sr25519::Signature::try_from(self.0.as_fixed_bytes().as_ref())
+			.map(|s| runtime_io::sr25519_verify(&s, msg.get(), &signer))
+			.unwrap_or(false)
+		|| ed25519::Signature::try_from(self.0.as_fixed_bytes().as_ref())
+			.and_then(|s| ed25519::Public::try_from(signer.0.as_ref()).map(|p| (s, p)))
+			.map(|(s, p)| runtime_io::ed25519_verify(&s, msg.get(), &p))
+			.unwrap_or(false)
 	}
 }
 

substrate/core/sr-primitives/src/testing.rs

@@ -20,46 +20,84 @@ use serde::{Serialize, Serializer, Deserialize, de::Error as DeError, Deserializ
 use std::{fmt::Debug, ops::Deref, fmt};
 use crate::codec::{Codec, Encode, Decode};
 use crate::traits::{
-	self, Checkable, Applyable, BlakeTwo256, OpaqueKeys, TypedKey, DispatchError, DispatchResult,
+	self, Checkable, Applyable, BlakeTwo256, OpaqueKeys, DispatchError, DispatchResult,
 	ValidateUnsigned, SignedExtension, Dispatchable,
 };
 use crate::{generic, KeyTypeId};
 use crate::weights::{GetDispatchInfo, DispatchInfo};
 pub use primitives::H256;
-use primitives::U256;
-use primitives::ed25519::{Public as AuthorityId};
+use primitives::{crypto::{CryptoType, Dummy, key_types, Public}, U256};
 use crate::transaction_validity::TransactionValidity;
 
 /// Authority Id
-#[derive(Default, PartialEq, Eq, Clone, Encode, Decode, Debug)]
-#[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
+#[derive(Default, PartialEq, Eq, Clone, Encode, Decode, Debug, Hash, Serialize, Deserialize)]
 pub struct UintAuthorityId(pub u64);
-impl Into<AuthorityId> for UintAuthorityId {
-	fn into(self) -> AuthorityId {
+
+impl UintAuthorityId {
+	/// Convert this authority id into a public key.
+	pub fn to_public_key<T: Public>(&self) -> T {
 		let bytes: [u8; 32] = U256::from(self.0).into();
-		AuthorityId(bytes)
+		T::from_slice(&bytes)
 	}
 }
 
-/// The key-type of the `UintAuthorityId`
-pub const UINT_DUMMY_KEY: KeyTypeId = 0xdeadbeef;
-
-impl TypedKey for UintAuthorityId {
-	const KEY_TYPE: KeyTypeId = UINT_DUMMY_KEY;
+impl CryptoType for UintAuthorityId {
+	type Pair = Dummy;
 }
 
 impl AsRef<[u8]> for UintAuthorityId {
 	fn as_ref(&self) -> &[u8] {
-		let ptr = self.0 as *const _;
-		// It's safe to do this here since `UintAuthorityId` is `u64`.
-		unsafe { std::slice::from_raw_parts(ptr, 8) }
+		unsafe {
+			std::slice::from_raw_parts(&self.0 as *const u64 as *const _, std::mem::size_of::<u64>())
+		}
+	}
+}
+
+impl app_crypto::RuntimeAppPublic for UintAuthorityId {
+	type Signature = u64;
+
+	fn all() -> Vec<Self> {
+		unimplemented!("`all()` not available for `UintAuthorityId`.")
+	}
+
+	#[cfg(feature = "std")]
+	fn generate_pair(_: Option<&str>) -> Self {
+		use rand::RngCore;
+		UintAuthorityId(rand::thread_rng().next_u64())
+	}
+
+	#[cfg(not(feature = "std"))]
+	fn generate_pair(_: Option<&str>) -> Self {
+		unimplemented!("`generate_pair` not implemented for `UIntAuthorityId` on `no_std`.")
+	}
+
+	fn sign<M: AsRef<[u8]>>(&self, msg: &M) -> Option<Self::Signature> {
+		let mut signature = [0u8; 8];
+		msg.as_ref().iter()
+			.chain(rstd::iter::repeat(&42u8))
+			.take(8)
+			.enumerate()
+			.for_each(|(i, v)| { signature[i] = *v; });
+
+		Some(u64::from_le_bytes(signature))
+	}
+
+	fn verify<M: AsRef<[u8]>>(&self, msg: &M, signature: &Self::Signature) -> bool {
+		let mut msg_signature = [0u8; 8];
+		msg.as_ref().iter()
+			.chain(rstd::iter::repeat(&42))
+			.take(8)
+			.enumerate()
+			.for_each(|(i, v)| { msg_signature[i] = *v; });
+
+		u64::from_le_bytes(msg_signature) == *signature
 	}
 }
 
 impl OpaqueKeys for UintAuthorityId {
 	type KeyTypeIds = std::iter::Cloned<std::slice::Iter<'static, KeyTypeId>>;
 
-	fn key_ids() -> Self::KeyTypeIds { [UINT_DUMMY_KEY].iter().cloned() }
+	fn key_ids() -> Self::KeyTypeIds { [key_types::DUMMY].iter().cloned() }
 	// Unsafe, i know, but it's test code and it's just there because it's really convenient to
 	// keep `UintAuthorityId` as a u64 under the hood.
 	fn get_raw(&self, _: KeyTypeId) -> &[u8] {
@@ -155,8 +193,7 @@ impl<Xt> traits::Extrinsic for ExtrinsicWrapper<Xt> {
 	}
 }
 
-impl<Xt: Encode> serde::Serialize for ExtrinsicWrapper<Xt>
-{
+impl<Xt: Encode> serde::Serialize for ExtrinsicWrapper<Xt> {
 	fn serialize<S>(&self, seq: S) -> Result<S::Ok, S::Error> where S: ::serde::Serializer {
 		self.using_encoded(|bytes| seq.serialize_bytes(bytes))
 	}
@@ -249,13 +286,13 @@ impl<Call: Codec + Sync + Send, Extra> traits::Extrinsic for TestXt<Call, Extra>
 
 impl<Origin, Call, Extra> Applyable for TestXt<Call, Extra> where
 	Call: 'static + Sized + Send + Sync + Clone + Eq + Codec + Debug + Dispatchable<Origin=Origin>,
-	Extra: SignedExtension<AccountId=u64>,
+	Extra: SignedExtension<AccountId=u64, Call=Call>,
 	Origin: From<Option<u64>>
 {
 	type AccountId = u64;
 	type Call = Call;
 
-	fn sender(&self) -> Option<&u64> { self.0.as_ref().map(|x| &x.0) }
+	fn sender(&self) -> Option<&Self::AccountId> { self.0.as_ref().map(|x| &x.0) }
 
 	/// Checks to see if this is a valid *transaction*. It returns information on it if so.
 	fn validate<U: ValidateUnsigned<Call=Self::Call>>(&self,
@@ -272,10 +309,10 @@ impl<Origin, Call, Extra> Applyable for TestXt<Call, Extra> where
 		len: usize,
 	) -> Result<DispatchResult, DispatchError> {
 		let maybe_who = if let Some((who, extra)) = self.0 {
-			Extra::pre_dispatch(extra, &who, info, len)?;
+			Extra::pre_dispatch(extra, &who, &self.1, info, len)?;
 			Some(who)
 		} else {
-			Extra::pre_dispatch_unsigned(info, len)?;
+			Extra::pre_dispatch_unsigned(&self.1, info, len)?;
 			None
 		};
 		Ok(self.1.dispatch(maybe_who.into()))

substrate/core/sr-primitives/src/traits.rs

@@ -26,7 +26,6 @@ use crate::codec::{Codec, Encode, Decode, HasCompact};
 use crate::transaction_validity::{ValidTransaction, TransactionValidity};
 use crate::generic::{Digest, DigestItem};
 use crate::weights::DispatchInfo;
-pub use primitives::crypto::TypedKey;
 pub use integer_sqrt::IntegerSquareRoot;
 pub use num_traits::{
 	Zero, One, Bounded, CheckedAdd, CheckedSub, CheckedMul, CheckedDiv,
@@ -60,14 +59,14 @@ pub trait Verify {
 impl Verify for primitives::ed25519::Signature {
 	type Signer = primitives::ed25519::Public;
 	fn verify<L: Lazy<[u8]>>(&self, mut msg: L, signer: &Self::Signer) -> bool {
-		runtime_io::ed25519_verify(self.as_ref(), msg.get(), signer)
+		runtime_io::ed25519_verify(self, msg.get(), signer)
 	}
 }
 
 impl Verify for primitives::sr25519::Signature {
 	type Signer = primitives::sr25519::Public;
 	fn verify<L: Lazy<[u8]>>(&self, mut msg: L, signer: &Self::Signer) -> bool {
-		runtime_io::sr25519_verify(self.as_ref(), msg.get(), signer)
+		runtime_io::sr25519_verify(self, msg.get(), signer)
 	}
 }
 
@@ -745,7 +744,7 @@ pub enum DispatchError {
 	Payment,
 
 	/// General error to do with the exhaustion of block resources.
-	Resource,
+	Exhausted,
 
 	/// General error to do with the permissions of the sender.
 	NoPermission,
@@ -761,16 +760,13 @@ pub enum DispatchError {
 
 	/// General error to do with the transaction's proofs (e.g. signature).
 	BadProof,
-
-/*	/// General error to do with actually executing the dispatched logic.
-	User(&'static str),*/
 }
 
 impl From<DispatchError> for i8 {
 	fn from(e: DispatchError) -> i8 {
 		match e {
 			DispatchError::Payment => -64,
-			DispatchError::Resource => -65,
+			DispatchError::Exhausted => -65,
 			DispatchError::NoPermission => -66,
 			DispatchError::BadState => -67,
 			DispatchError::Stale => -68,
@@ -805,6 +801,9 @@ pub trait SignedExtension:
 	/// The type which encodes the sender identity.
 	type AccountId;
 
+	/// The type which encodes the call to be dispatched.
+	type Call;
+
 	/// Any additional data that will go into the signed payload. This may be created dynamically
 	/// from the transaction using the `additional_signed` function.
 	type AdditionalSigned: Encode;
@@ -813,35 +812,45 @@ pub trait SignedExtension:
 	/// also perform any pre-signature-verification checks and return an error if needed.
 	fn additional_signed(&self) -> Result<Self::AdditionalSigned, &'static str>;
 
-		/// Validate a signed transaction for the transaction queue.
+	/// Validate a signed transaction for the transaction queue.
 	fn validate(
 		&self,
 		_who: &Self::AccountId,
+		_call: &Self::Call,
 		_info: DispatchInfo,
 		_len: usize,
-	) -> Result<ValidTransaction, DispatchError> { Ok(Default::default()) }
+	) -> Result<ValidTransaction, DispatchError> {
+		Ok(Default::default())
+	}
 
 	/// Do any pre-flight stuff for a signed transaction.
 	fn pre_dispatch(
 		self,
 		who: &Self::AccountId,
+		call: &Self::Call,
 		info: DispatchInfo,
 		len: usize,
-	) -> Result<(), DispatchError> { self.validate(who, info, len).map(|_| ()) }
+	) -> Result<(), DispatchError> {
+		self.validate(who, call, info, len).map(|_| ())
+	}
 
 	/// Validate an unsigned transaction for the transaction queue. Normally the default
 	/// implementation is fine since `ValidateUnsigned` is a better way of recognising and
 	/// validating unsigned transactions.
 	fn validate_unsigned(
+		_call: &Self::Call,
 		_info: DispatchInfo,
 		_len: usize,
 	) -> Result<ValidTransaction, DispatchError> { Ok(Default::default()) }
 
 	/// Do any pre-flight stuff for a unsigned transaction.
 	fn pre_dispatch_unsigned(
+		call: &Self::Call,
 		info: DispatchInfo,
 		len: usize,
-	) -> Result<(), DispatchError> { Self::validate_unsigned(info, len).map(|_| ()) }
+	) -> Result<(), DispatchError> {
+		Self::validate_unsigned(call, info, len).map(|_| ())
+	}
 }
 
 macro_rules! tuple_impl_indexed {
@@ -851,9 +860,11 @@ macro_rules! tuple_impl_indexed {
 	([$($direct:ident)+] ; [$($index:tt,)+]) => {
 		impl<
 			AccountId,
-			$($direct: SignedExtension<AccountId=AccountId>),+
+			Call,
+			$($direct: SignedExtension<AccountId=AccountId, Call=Call>),+
 		> SignedExtension for ($($direct),+,) {
 			type AccountId = AccountId;
+			type Call = Call;
 			type AdditionalSigned = ($($direct::AdditionalSigned,)+);
 			fn additional_signed(&self) -> Result<Self::AdditionalSigned, &'static str> {
 				Ok(( $(self.$index.additional_signed()?,)+ ))
@@ -861,33 +872,37 @@ macro_rules! tuple_impl_indexed {
 			fn validate(
 				&self,
 				who: &Self::AccountId,
+				call: &Self::Call,
 				info: DispatchInfo,
 				len: usize,
 			) -> Result<ValidTransaction, DispatchError> {
-				let aggregator = vec![$(<$direct as SignedExtension>::validate(&self.$index, who, info, len)?),+];
+				let aggregator = vec![$(<$direct as SignedExtension>::validate(&self.$index, who, call, info, len)?),+];
 				Ok(aggregator.into_iter().fold(ValidTransaction::default(), |acc, a| acc.combine_with(a)))
 			}
 			fn pre_dispatch(
 				self,
 				who: &Self::AccountId,
+				call: &Self::Call,
 				info: DispatchInfo,
 				len: usize,
 			) -> Result<(), DispatchError> {
-				$(self.$index.pre_dispatch(who, info, len)?;)+
+				$(self.$index.pre_dispatch(who, call, info, len)?;)+
 				Ok(())
 			}
 			fn validate_unsigned(
+				call: &Self::Call,
 				info: DispatchInfo,
 				len: usize,
 			) -> Result<ValidTransaction, DispatchError> {
-				let aggregator = vec![$($direct::validate_unsigned(info, len)?),+];
+				let aggregator = vec![$($direct::validate_unsigned(call, info, len)?),+];
 				Ok(aggregator.into_iter().fold(ValidTransaction::default(), |acc, a| acc.combine_with(a)))
 			}
 			fn pre_dispatch_unsigned(
+				call: &Self::Call,
 				info: DispatchInfo,
 				len: usize,
 			) -> Result<(), DispatchError> {
-				$($direct::pre_dispatch_unsigned(info, len)?;)+
+				$($direct::pre_dispatch_unsigned(call, info, len)?;)+
 				Ok(())
 			}
 		}
@@ -910,11 +925,12 @@ macro_rules! tuple_impl_indexed {
 #[allow(non_snake_case)]
 tuple_impl_indexed!(A, B, C, D, E, F, G, H, I, J, ; 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,);
 
-/// Only for base bone testing when you don't care about signed extensions at all.\
+/// Only for bare bone testing when you don't care about signed extensions at all.
 #[cfg(feature = "std")]
 impl SignedExtension for () {
 	type AccountId = u64;
 	type AdditionalSigned = ();
+	type Call = ();
 	fn additional_signed(&self) -> rstd::result::Result<(), &'static str> { Ok(()) }
 }
 
@@ -1208,14 +1224,14 @@ macro_rules! count {
 /// just the bytes of the key.
 ///
 /// ```rust
-/// use sr_primitives::{impl_opaque_keys, key_types, KeyTypeId};
+/// use sr_primitives::{impl_opaque_keys, key_types, KeyTypeId, app_crypto::{sr25519, ed25519}};
 ///
 /// impl_opaque_keys! {
 /// 	pub struct Keys {
 /// 		#[id(key_types::ED25519)]
-/// 		pub ed25519: [u8; 32],
+/// 		pub ed25519: ed25519::AppPublic,
 /// 		#[id(key_types::SR25519)]
-/// 		pub sr25519: [u8; 32],
+/// 		pub sr25519: sr25519::AppPublic,
 /// 	}
 /// }
 /// ```
@@ -1237,22 +1253,34 @@ macro_rules! impl_opaque_keys {
 			)*
 		}
 
+		impl $name {
+			/// Generate a set of keys with optionally using the given seed.
+			///
+			/// The generated key pairs are stored in the keystore.
+			///
+			/// Returns the concatenated SCALE encoded public keys.
+			pub fn generate(seed: Option<&str>) -> $crate::rstd::vec::Vec<u8> {
+				let keys = Self{
+					$(
+						$field: <$type as $crate::app_crypto::RuntimeAppPublic>::generate_pair(seed),
+					)*
+				};
+				$crate::codec::Encode::encode(&keys)
+			}
+		}
+
 		impl $crate::traits::OpaqueKeys for $name {
 			type KeyTypeIds = $crate::rstd::iter::Cloned<
 				$crate::rstd::slice::Iter<'static, $crate::KeyTypeId>
 			>;
 
 			fn key_ids() -> Self::KeyTypeIds {
-				[
-					$($key_id),*
-				].iter().cloned()
+				[ $($key_id),* ].iter().cloned()
 			}
 
 			fn get_raw(&self, i: $crate::KeyTypeId) -> &[u8] {
 				match i {
-					$(
-						i if i == $key_id => self.$field.as_ref(),
-					)*
+					$( i if i == $key_id => self.$field.as_ref(), )*
 					_ => &[],
 				}
 			}