Overhaul crypto (Schnorr/Ristretto, HDKD, BIP39) (#1795)

* Rijig to Ristretto

* Rebuild wasm

* adds compatibility test with the wasm module

* Add Ed25519-BIP39 support

* Bump subkey version

* Update CLI output

* New keys.

* Standard phrase/password/path keys.

* Subkey uses S-URI for secrets

* Move everything to use new HDKD crypto.

* Test fixes

* Ignore old test vector.

* fix the ^^ old test vector.

* Fix tests

* Test fixes

* Cleanups

* Fix broken key conversion logic in grandpa

CC @rphmeier

* Remove legacy Keyring usage

* Traitify `Pair`

* Replace Ed25519AuthorityId with ed25519::Public

* Expunge Ed25519AuthorityId type!

* Replace Sr25519AuthorityId with sr25519::Public

* Remove dodgy crypto type-punning conversions

* Fix some tests

* Avoid trait

* Deduplicate DeriveJunction string decode

* Remove cruft code

* Fix test

* Minor removals

* Build fix

* Subkey supports sign and verify

* Inspect works for public key URIs

* Remove more crypto type-punning

* Fix typo

* Fix tests

substrate/core/sr-primitives/src/generic/digest.rs

@@ -24,8 +24,6 @@ use rstd::prelude::*;
 use crate::codec::{Decode, Encode, Codec, Input};
 use crate::traits::{self, Member, DigestItem as DigestItemT, MaybeHash};
 
-use substrate_primitives::hash::H512 as Signature;
-
 /// Generic header digest.
 #[derive(PartialEq, Eq, Clone, Encode, Decode)]
 #[cfg_attr(feature = "std", derive(Debug, Serialize))]
@@ -63,7 +61,7 @@ impl<Item> traits::Digest for Digest<Item> where
 /// provide opaque access to other items.
 #[derive(PartialEq, Eq, Clone)]
 #[cfg_attr(feature = "std", derive(Debug))]
-pub enum DigestItem<Hash, AuthorityId> {
+pub enum DigestItem<Hash, AuthorityId, SealSignature> {
 	/// System digest item announcing that authorities set has been changed
 	/// in the block. Contains the new set of authorities.
 	AuthoritiesChange(Vec<AuthorityId>),
@@ -72,13 +70,13 @@ pub enum DigestItem<Hash, AuthorityId> {
 	/// trie creation.
 	ChangesTrieRoot(Hash),
 	/// Put a Seal on it
-	Seal(u64, Signature),
+	Seal(u64, SealSignature),
 	/// Any 'non-system' digest item, opaque to the native code.
 	Other(Vec<u8>),
 }
 
 #[cfg(feature = "std")]
-impl<Hash: Encode, AuthorityId: Encode> ::serde::Serialize for DigestItem<Hash, AuthorityId> {
+impl<Hash: Encode, AuthorityId: Encode, SealSignature: Encode> ::serde::Serialize for DigestItem<Hash, AuthorityId, SealSignature> {
 	fn serialize<S>(&self, seq: S) -> Result<S::Ok, S::Error> where S: ::serde::Serializer {
 		self.using_encoded(|bytes| {
 			::substrate_primitives::bytes::serialize(bytes, seq)
@@ -91,13 +89,13 @@ impl<Hash: Encode, AuthorityId: Encode> ::serde::Serialize for DigestItem<Hash,
 /// final runtime implementations for encoding/decoding its log items.
 #[derive(PartialEq, Eq, Clone)]
 #[cfg_attr(feature = "std", derive(Debug))]
-pub enum DigestItemRef<'a, Hash: 'a, AuthorityId: 'a> {
+pub enum DigestItemRef<'a, Hash: 'a, AuthorityId: 'a, SealSignature: 'a> {
 	/// Reference to `DigestItem::AuthoritiesChange`.
 	AuthoritiesChange(&'a [AuthorityId]),
 	/// Reference to `DigestItem::ChangesTrieRoot`.
 	ChangesTrieRoot(&'a Hash),
 	/// A sealed signature for testing
-	Seal(&'a u64, &'a Signature),
+	Seal(&'a u64, &'a SealSignature),
 	/// Any 'non-system' digest item, opaque to the native code.
 	/// Reference to `DigestItem::Other`.
 	Other(&'a Vec<u8>),
@@ -116,7 +114,7 @@ enum DigestItemType {
 	Seal,
 }
 
-impl<Hash, AuthorityId> DigestItem<Hash, AuthorityId> {
+impl<Hash, AuthorityId, SealSignature> DigestItem<Hash, AuthorityId, SealSignature> {
 	/// Returns Some if `self` is a `DigestItem::Other`.
 	pub fn as_other(&self) -> Option<&Vec<u8>> {
 		match *self {
@@ -126,7 +124,7 @@ impl<Hash, AuthorityId> DigestItem<Hash, AuthorityId> {
 	}
 
 	/// Returns a 'referencing view' for this digest item.
-	fn dref<'a>(&'a self) -> DigestItemRef<'a, Hash, AuthorityId> {
+	fn dref<'a>(&'a self) -> DigestItemRef<'a, Hash, AuthorityId, SealSignature> {
 		match *self {
 			DigestItem::AuthoritiesChange(ref v) => DigestItemRef::AuthoritiesChange(v),
 			DigestItem::ChangesTrieRoot(ref v) => DigestItemRef::ChangesTrieRoot(v),
@@ -139,7 +137,8 @@ impl<Hash, AuthorityId> DigestItem<Hash, AuthorityId> {
 impl<
 	Hash: Codec + Member,
 	AuthorityId: Codec + Member + MaybeHash,
-> traits::DigestItem for DigestItem<Hash, AuthorityId> {
+	SealSignature: Codec + Member,
+> traits::DigestItem for DigestItem<Hash, AuthorityId, SealSignature> {
 	type Hash = Hash;
 	type AuthorityId = AuthorityId;
 
@@ -152,13 +151,13 @@ impl<
 	}
 }
 
-impl<Hash: Encode, AuthorityId: Encode> Encode for DigestItem<Hash, AuthorityId> {
+impl<Hash: Encode, AuthorityId: Encode, SealSignature: Encode> Encode for DigestItem<Hash, AuthorityId, SealSignature> {
 	fn encode(&self) -> Vec<u8> {
 		self.dref().encode()
 	}
 }
 
-impl<Hash: Decode, AuthorityId: Decode> Decode for DigestItem<Hash, AuthorityId> {
+impl<Hash: Decode, AuthorityId: Decode, SealSignature: Decode> Decode for DigestItem<Hash, AuthorityId, SealSignature> {
 	fn decode<I: Input>(input: &mut I) -> Option<Self> {
 		let item_type: DigestItemType = Decode::decode(input)?;
 		match item_type {
@@ -169,7 +168,7 @@ impl<Hash: Decode, AuthorityId: Decode> Decode for DigestItem<Hash, AuthorityId>
 				Decode::decode(input)?,
 			)),
 			DigestItemType::Seal => {
-				let vals: (u64, Signature) = Decode::decode(input)?;
+				let vals: (u64, SealSignature) = Decode::decode(input)?;
 				Some(DigestItem::Seal(vals.0, vals.1))
 			},
 			DigestItemType::Other => Some(DigestItem::Other(
@@ -179,7 +178,7 @@ impl<Hash: Decode, AuthorityId: Decode> Decode for DigestItem<Hash, AuthorityId>
 	}
 }
 
-impl<'a, Hash: Codec + Member, AuthorityId: Codec + Member> DigestItemRef<'a, Hash, AuthorityId> {
+impl<'a, Hash: Codec + Member, AuthorityId: Codec + Member, SealSignature: Codec + Member> DigestItemRef<'a, Hash, AuthorityId, SealSignature> {
 	/// Cast this digest item into `AuthoritiesChange`.
 	pub fn as_authorities_change(&self) -> Option<&'a [AuthorityId]> {
 		match *self {
@@ -197,7 +196,7 @@ impl<'a, Hash: Codec + Member, AuthorityId: Codec + Member> DigestItemRef<'a, Ha
 	}
 }
 
-impl<'a, Hash: Encode, AuthorityId: Encode> Encode for DigestItemRef<'a, Hash, AuthorityId> {
+impl<'a, Hash: Encode, AuthorityId: Encode, SealSignature: Encode> Encode for DigestItemRef<'a, Hash, AuthorityId, SealSignature> {
 	fn encode(&self) -> Vec<u8> {
 		let mut v = Vec::new();
 
@@ -227,6 +226,7 @@ impl<'a, Hash: Encode, AuthorityId: Encode> Encode for DigestItemRef<'a, Hash, A
 #[cfg(test)]
 mod tests {
 	use super::*;
+	use substrate_primitives::hash::H512 as Signature;
 
 	#[test]
 	fn should_serialize_digest() {

substrate/core/sr-primitives/src/generic/tests.rs

@@ -17,12 +17,12 @@
 //! Tests for the generic implementations of Extrinsic/Header/Block.
 
 use crate::codec::{Decode, Encode};
-use substrate_primitives::H256;
+use substrate_primitives::{H256, H512};
 use super::DigestItem;
 
 #[test]
 fn system_digest_item_encoding() {
-	let item = DigestItem::AuthoritiesChange::<H256, u32>(vec![10, 20, 30]);
+	let item = DigestItem::AuthoritiesChange::<H256, u32, H512>(vec![10, 20, 30]);
 	let encoded = item.encode();
 	assert_eq!(encoded, vec![
 		// type = DigestItemType::AuthoritiesChange
@@ -35,13 +35,13 @@ fn system_digest_item_encoding() {
 		30, 0, 0, 0,
 	]);
 
-	let decoded: DigestItem<H256, u32> = Decode::decode(&mut &encoded[..]).unwrap();
+	let decoded: DigestItem<H256, u32, H512> = Decode::decode(&mut &encoded[..]).unwrap();
 	assert_eq!(item, decoded);
 }
 
 #[test]
 fn non_system_digest_item_encoding() {
-	let item = DigestItem::Other::<H256, u32>(vec![10, 20, 30]);
+	let item = DigestItem::Other::<H256, u32, H512>(vec![10, 20, 30]);
 	let encoded = item.encode();
 	assert_eq!(encoded, vec![
 		// type = DigestItemType::Other
@@ -52,6 +52,6 @@ fn non_system_digest_item_encoding() {
 		10, 20, 30,
 	]);
 
-	let decoded: DigestItem<H256, u32> = Decode::decode(&mut &encoded[..]).unwrap();
+	let decoded: DigestItem<H256, u32, H512> = Decode::decode(&mut &encoded[..]).unwrap();
 	assert_eq!(item, decoded);
 }

substrate/core/sr-primitives/src/lib.rs

@@ -30,7 +30,7 @@ pub use serde_derive;
 pub use runtime_io::{StorageOverlay, ChildrenStorageOverlay};
 
 use rstd::prelude::*;
-use substrate_primitives::hash::{H256, H512};
+use substrate_primitives::{ed25519, sr25519, hash::{H256, H512}};
 use codec::{Encode, Decode};
 
 #[cfg(feature = "std")]
@@ -251,39 +251,64 @@ impl From<codec::Compact<Perbill>> for Perbill {
 	}
 }
 
-/// Ed25519 signature verify.
+/// Signature verify that can work with any known signature types..
+#[derive(Eq, PartialEq, Clone, Encode, Decode)]
+#[cfg_attr(feature = "std", derive(Debug))]
+pub enum MultiSignature {
+	/// An Ed25519 signature.
+	Ed25519(ed25519::Signature),
+	/// An Sr25519 signature.
+	Sr25519(sr25519::Signature),
+}
+
+impl Default for MultiSignature {
+	fn default() -> Self {
+		MultiSignature::Ed25519(Default::default())
+	}
+}
+
+/// Public key for any known crypto algorithm.
+#[derive(Eq, PartialEq, Ord, PartialOrd, Clone, Encode, Decode)]
+#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))]
+pub enum MultiSigner {
+	/// An Ed25519 identity.
+	Ed25519(ed25519::Public),
+	/// An Sr25519 identity.
+	Sr25519(sr25519::Public),
+}
+
+impl Default for MultiSigner {
+	fn default() -> Self {
+		MultiSigner::Ed25519(Default::default())
+	}
+}
+
+impl Verify for MultiSignature {
+	type Signer = MultiSigner;
+	fn verify<L: Lazy<[u8]>>(&self, msg: L, signer: &Self::Signer) -> bool {
+		match (self, signer) {
+			(MultiSignature::Ed25519(ref sig), &MultiSigner::Ed25519(ref who)) => sig.verify(msg, who),
+			(MultiSignature::Sr25519(ref sig), &MultiSigner::Sr25519(ref who)) => sig.verify(msg, who),
+			_ => false,
+		}
+	}
+}
+
+/// Signature verify that can work with any known signature types..
 #[derive(Eq, PartialEq, Clone, Default, Encode, Decode)]
 #[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))]
-pub struct Ed25519Signature(pub H512);
+pub struct AnySignature(H512);
 
-impl Verify for Ed25519Signature {
-	type Signer = H256;
-	fn verify<L: Lazy<[u8]>>(&self, mut msg: L, signer: &Self::Signer) -> bool {
-		runtime_io::ed25519_verify((self.0).as_fixed_bytes(), msg.get(), &signer.as_bytes())
-	}
-}
-
-impl From<H512> for Ed25519Signature {
-	fn from(h: H512) -> Ed25519Signature {
-		Ed25519Signature(h)
-	}
-}
-
-/// Sr25519 signature verify.
-#[derive(Eq, PartialEq, Clone, Default, Encode, Decode)]
+/// Public key for any known crypto algorithm.
+#[derive(Eq, PartialEq, Ord, PartialOrd, Clone, Default, Encode, Decode)]
 #[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))]
-pub struct Sr25519Signature(pub H512);
+pub struct AnySigner(H256);
 
-impl Verify for Sr25519Signature {
-	type Signer = H256;
-	fn verify<L: Lazy<[u8]>>(&self, mut msg: L, signer: &Self::Signer) -> bool {
-		runtime_io::sr25519_verify((self.0).as_fixed_bytes(), msg.get(), &signer.as_bytes())
-	}
-}
-
-impl From<H512> for Sr25519Signature {
-	fn from(h: H512) -> Sr25519Signature {
-		Sr25519Signature(h)
+impl Verify for AnySignature {
+	type Signer = AnySigner;
+	fn verify<L: Lazy<[u8]>>(&self, mut msg: L, signer: &AnySigner) -> bool {
+		runtime_io::sr25519_verify(self.0.as_fixed_bytes(), msg.get(), &signer.0.as_bytes()) ||
+			runtime_io::ed25519_verify(self.0.as_fixed_bytes(), msg.get(), &signer.0.as_bytes())
 	}
 }
 
@@ -581,7 +606,7 @@ impl traits::Extrinsic for OpaqueExtrinsic {
 
 #[cfg(test)]
 mod tests {
-	use substrate_primitives::hash::H256;
+	use substrate_primitives::hash::{H256, H512};
 	use crate::codec::{Encode, Decode};
 	use crate::traits::DigestItem;
 
@@ -616,7 +641,7 @@ mod tests {
 	}
 
 	impl_outer_log! {
-		pub enum Log(InternalLog: DigestItem<H256, u64>) for Runtime {
+		pub enum Log(InternalLog: DigestItem<H256, u64, H512>) for Runtime {
 			a(AuthoritiesChange), b()
 		}
 	}
@@ -636,16 +661,16 @@ mod tests {
 		assert_eq!(auth_change, decoded_auth_change);
 
 		// interpret regular item using `generic::DigestItem`
-		let generic_b1: super::generic::DigestItem<H256, u64> = Decode::decode(&mut &encoded_b1[..]).unwrap();
+		let generic_b1: super::generic::DigestItem<H256, u64, H512> = Decode::decode(&mut &encoded_b1[..]).unwrap();
 		match generic_b1 {
 			super::generic::DigestItem::Other(_) => (),
 			_ => panic!("unexpected generic_b1: {:?}", generic_b1),
 		}
 
 		// interpret system item using `generic::DigestItem`
-		let generic_auth_change: super::generic::DigestItem<H256, u64> = Decode::decode(&mut &encoded_auth_change[..]).unwrap();
+		let generic_auth_change: super::generic::DigestItem<H256, u64, H512> = Decode::decode(&mut &encoded_auth_change[..]).unwrap();
 		match generic_auth_change {
-			super::generic::DigestItem::AuthoritiesChange::<H256, u64>(authorities) => assert_eq!(authorities, vec![100, 200, 300]),
+			super::generic::DigestItem::AuthoritiesChange::<H256, u64, H512>(authorities) => assert_eq!(authorities, vec![100, 200, 300]),
 			_ => panic!("unexpected generic_auth_change: {:?}", generic_auth_change),
 		}
 

substrate/core/sr-primitives/src/testing.rs

@@ -24,30 +24,30 @@ use std::{fmt::Debug, ops::Deref, fmt};
 use crate::codec::{Codec, Encode, Decode};
 use crate::traits::{self, Checkable, Applyable, BlakeTwo256, Convert};
 use crate::generic::DigestItem as GenDigestItem;
-
-pub use substrate_primitives::{H256, Ed25519AuthorityId};
+pub use substrate_primitives::H256;
 use substrate_primitives::U256;
+use substrate_primitives::ed25519::{Public as AuthorityId, Signature as AuthoritySignature};
 
 /// Authority Id
 #[derive(Default, PartialEq, Eq, Clone, Encode, Decode, Debug)]
 #[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
 pub struct UintAuthorityId(pub u64);
-impl Into<Ed25519AuthorityId> for UintAuthorityId {
-	fn into(self) -> Ed25519AuthorityId {
+impl Into<AuthorityId> for UintAuthorityId {
+	fn into(self) -> AuthorityId {
 		let bytes: [u8; 32] = U256::from(self.0).into();
-		Ed25519AuthorityId(bytes)
+		AuthorityId(bytes)
 	}
 }
 
 /// Converter between u64 and the AuthorityId wrapper type.
 pub struct ConvertUintAuthorityId;
-impl Convert<u64, UintAuthorityId> for ConvertUintAuthorityId {
-	fn convert(a: u64) -> UintAuthorityId {
-		UintAuthorityId(a)
+impl Convert<u64, Option<UintAuthorityId>> for ConvertUintAuthorityId {
+	fn convert(a: u64) -> Option<UintAuthorityId> {
+		Some(UintAuthorityId(a))
 	}
 }
 /// Digest item
-pub type DigestItem = GenDigestItem<H256, Ed25519AuthorityId>;
+pub type DigestItem = GenDigestItem<H256, AuthorityId, AuthoritySignature>;
 
 /// Header Digest
 #[derive(Default, PartialEq, Eq, Clone, Serialize, Debug, Encode, Decode)]

substrate/core/sr-primitives/src/traits.rs

@@ -55,6 +55,20 @@ pub trait Verify {
 	fn verify<L: Lazy<[u8]>>(&self, msg: L, signer: &Self::Signer) -> bool;
 }
 
+impl Verify for substrate_primitives::ed25519::Signature {
+	type Signer = substrate_primitives::ed25519::Public;
+	fn verify<L: Lazy<[u8]>>(&self, mut msg: L, signer: &Self::Signer) -> bool {
+		runtime_io::ed25519_verify(self.as_ref(), msg.get(), signer)
+	}
+}
+
+impl Verify for substrate_primitives::sr25519::Signature {
+	type Signer = substrate_primitives::sr25519::Public;
+	fn verify<L: Lazy<[u8]>>(&self, mut msg: L, signer: &Self::Signer) -> bool {
+		runtime_io::sr25519_verify(self.as_ref(), msg.get(), signer)
+	}
+}
+
 /// Some sort of check on the origin is performed by this object.
 pub trait EnsureOrigin<OuterOrigin> {
 	/// A return type.
@@ -134,6 +148,16 @@ pub trait Convert<A, B> {
 	fn convert(a: A) -> B;
 }
 
+impl<A, B: Default> Convert<A, B> for () {
+	fn convert(_: A) -> B { Default::default() }
+}
+
+/// A structure that performs identity conversion.
+pub struct Identity;
+impl<T> Convert<T, T> for Identity {
+	fn convert(a: T) -> T { a }
+}
+
 /// Simple trait similar to `Into`, except that it can be used to convert numerics between each
 /// other.
 pub trait As<T> {
@@ -161,15 +185,6 @@ macro_rules! impl_numerics {
 
 impl_numerics!(u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize);
 
-/// A structure that performs identity conversion.
-pub struct Identity;
-impl<T> Convert<T, T> for Identity {
-	fn convert(a: T) -> T { a }
-}
-impl<T> Convert<T, ()> for () {
-	fn convert(_: T) -> () { () }
-}
-
 /// A meta trait for arithmetic.
 pub trait SimpleArithmetic:
 	Zero + One + IntegerSquareRoot + As<u64> +