Overhaul crypto (Schnorr/Ristretto, HDKD, BIP39) (#1795)

* Rijig to Ristretto

* Rebuild wasm

* adds compatibility test with the wasm module

* Add Ed25519-BIP39 support

* Bump subkey version

* Update CLI output

* New keys.

* Standard phrase/password/path keys.

* Subkey uses S-URI for secrets

* Move everything to use new HDKD crypto.

* Test fixes

* Ignore old test vector.

* fix the ^^ old test vector.

* Fix tests

* Test fixes

* Cleanups

* Fix broken key conversion logic in grandpa

CC @rphmeier

* Remove legacy Keyring usage

* Traitify `Pair`

* Replace Ed25519AuthorityId with ed25519::Public

* Expunge Ed25519AuthorityId type!

* Replace Sr25519AuthorityId with sr25519::Public

* Remove dodgy crypto type-punning conversions

* Fix some tests

* Avoid trait

* Deduplicate DeriveJunction string decode

* Remove cruft code

* Fix test

* Minor removals

* Build fix

* Subkey supports sign and verify

* Inspect works for public key URIs

* Remove more crypto type-punning

* Fix typo

* Fix tests

substrate/core/consensus/aura/src/lib.rs

@@ -41,7 +41,7 @@ use runtime_primitives::{generic, generic::BlockId, Justification};
 use runtime_primitives::traits::{
 	Block, Header, Digest, DigestItemFor, DigestItem, ProvideRuntimeApi
 };
-use primitives::{Ed25519AuthorityId, ed25519};
+use primitives::{ed25519, Pair};
 use inherents::{InherentDataProviders, InherentData, RuntimeString};
 
 use futures::{Stream, Future, IntoFuture, future};
@@ -60,6 +60,9 @@ pub use aura_slots::SlotDuration;
 pub use aura_primitives::*;
 pub use consensus_common::SyncOracle;
 
+type AuthorityId = ed25519::Public;
+type Signature = ed25519::Signature;
+
 /// A handle to the network. This is generally implemented by providing some
 /// handle to a gossip service or similar.
 ///
@@ -73,16 +76,17 @@ pub trait Network: Clone {
 }
 
 /// Get slot author for given block along with authorities.
-fn slot_author(slot_num: u64, authorities: &[Ed25519AuthorityId]) -> Option<Ed25519AuthorityId> {
+fn slot_author(slot_num: u64, authorities: &[AuthorityId]) -> Option<AuthorityId> {
 	if authorities.is_empty() { return None }
 
 	let idx = slot_num % (authorities.len() as u64);
 	assert!(idx <= usize::max_value() as u64,
 		"It is impossible to have a vector with length beyond the address space; qed");
 
-	let current_author = *authorities.get(idx as usize)
+	let current_author = authorities.get(idx as usize)
 		.expect("authorities not empty; index constrained to list length;\
-				this is a valid index; qed");
+				this is a valid index; qed")
+		.clone();
 
 	Some(current_author)
 }
@@ -109,22 +113,22 @@ fn inherent_to_common_error(err: RuntimeString) -> consensus_common::Error {
 pub trait CompatibleDigestItem: Sized {
 	/// Construct a digest item which is a slot number and a signature on the
 	/// hash.
-	fn aura_seal(slot_number: u64, signature: ed25519::Signature) -> Self;
+	fn aura_seal(slot_number: u64, signature: Signature) -> Self;
 
 	/// If this item is an Aura seal, return the slot number and signature.
-	fn as_aura_seal(&self) -> Option<(u64, &ed25519::Signature)>;
+	fn as_aura_seal(&self) -> Option<(u64, Signature)>;
 }
 
-impl<Hash, AuthorityId> CompatibleDigestItem for generic::DigestItem<Hash, AuthorityId> {
+impl<Hash, AuthorityId> CompatibleDigestItem for generic::DigestItem<Hash, AuthorityId, Signature> {
 	/// Construct a digest item which is a slot number and a signature on the
 	/// hash.
-	fn aura_seal(slot_number: u64, signature: ed25519::Signature) -> Self {
+	fn aura_seal(slot_number: u64, signature: Signature) -> Self {
 		generic::DigestItem::Seal(slot_number, signature)
 	}
 	/// If this item is an Aura seal, return the slot number and signature.
-	fn as_aura_seal(&self) -> Option<(u64, &ed25519::Signature)> {
+	fn as_aura_seal(&self) -> Option<(u64, Signature)> {
 		match self {
-			generic::DigestItem::Seal(slot, ref sign) => Some((*slot, sign)),
+			generic::DigestItem::Seal(slot, ref sig) => Some((*slot, sig.clone().into())),
 			_ => None
 		}
 	}
@@ -162,7 +166,7 @@ pub fn start_aura_thread<B, C, E, I, SO, Error, OnExit>(
 	Error: From<C::Error> + From<I::Error> + 'static,
 	SO: SyncOracle + Send + Sync + Clone + 'static,
 	OnExit: Future<Item=(), Error=()> + Send + 'static,
-	DigestItemFor<B>: CompatibleDigestItem + DigestItem<AuthorityId=Ed25519AuthorityId> + 'static,
+	DigestItemFor<B>: CompatibleDigestItem + DigestItem<AuthorityId=AuthorityId> + 'static,
 	Error: ::std::error::Error + Send + From<::consensus_common::Error> + 'static,
 {
 	let worker = AuraWorker {
@@ -198,7 +202,7 @@ pub fn start_aura<B, C, E, I, SO, Error, OnExit>(
 	I: BlockImport<B> + Send + Sync + 'static,
 	Error: From<C::Error> + From<I::Error>,
 	SO: SyncOracle + Send + Sync + Clone,
-	DigestItemFor<B>: CompatibleDigestItem + DigestItem<AuthorityId=Ed25519AuthorityId>,
+	DigestItemFor<B>: CompatibleDigestItem + DigestItem<AuthorityId=AuthorityId>,
 	Error: ::std::error::Error + Send + 'static + From<::consensus_common::Error>,
 	OnExit: Future<Item=(), Error=()>,
 {
@@ -232,7 +236,7 @@ impl<B: Block, C, E, I, Error, SO> SlotWorker<B> for AuraWorker<C, E, I, SO> whe
 	I: BlockImport<B> + Send + Sync + 'static,
 	Error: From<C::Error> + From<I::Error>,
 	SO: SyncOracle + Send + Clone,
-	DigestItemFor<B>: CompatibleDigestItem + DigestItem<AuthorityId=Ed25519AuthorityId>,
+	DigestItemFor<B>: CompatibleDigestItem + DigestItem<AuthorityId=AuthorityId>,
 	Error: ::std::error::Error + Send + 'static + From<::consensus_common::Error>,
 {
 	type OnSlot = Box<Future<Item=(), Error=consensus_common::Error> + Send>;
@@ -387,7 +391,7 @@ impl<B: Block, C, E, I, Error, SO> SlotWorker<B> for AuraWorker<C, E, I, SO> whe
 /// if it's successful, returns the pre-header, the slot number, and the signat.
 //
 // FIXME #1018 needs misbehavior types
-fn check_header<B: Block>(slot_now: u64, mut header: B::Header, hash: B::Hash, authorities: &[Ed25519AuthorityId])
+fn check_header<B: Block>(slot_now: u64, mut header: B::Header, hash: B::Hash, authorities: &[AuthorityId])
 	-> Result<CheckedHeader<B::Header, ed25519::Signature>, String>
 	where DigestItemFor<B>: CompatibleDigestItem
 {
@@ -395,7 +399,7 @@ fn check_header<B: Block>(slot_now: u64, mut header: B::Header, hash: B::Hash, a
 		Some(x) => x,
 		None => return Err(format!("Header {:?} is unsealed", hash)),
 	};
-	let (slot_num, &sig) = match digest_item.as_aura_seal() {
+	let (slot_num, sig) = match digest_item.as_aura_seal() {
 		Some(x) => x,
 		None => return Err(format!("Header {:?} is unsealed", hash)),
 	};
@@ -416,7 +420,7 @@ fn check_header<B: Block>(slot_now: u64, mut header: B::Header, hash: B::Hash, a
 		let to_sign = (slot_num, pre_hash).encode();
 		let public = ed25519::Public(expected_author.0);
 
-		if ed25519::verify_strong(&sig, &to_sign[..], public) {
+		if ed25519::Pair::verify(&sig, &to_sign[..], public) {
 			Ok(CheckedHeader::Checked(header, slot_num, sig))
 		} else {
 			Err(format!("Bad signature on {:?}", hash))
@@ -555,7 +559,7 @@ impl<B: Block> ExtraVerification<B> for NothingExtra {
 impl<B: Block, C, E> Verifier<B> for AuraVerifier<C, E> where
 	C: Authorities<B> + ProvideRuntimeApi + Send + Sync,
 	C::Api: BlockBuilderApi<B>,
-	DigestItemFor<B>: CompatibleDigestItem + DigestItem<AuthorityId=Ed25519AuthorityId>,
+	DigestItemFor<B>: CompatibleDigestItem + DigestItem<AuthorityId=AuthorityId>,
 	E: ExtraVerification<B>,
 {
 	fn verify(
@@ -564,7 +568,7 @@ impl<B: Block, C, E> Verifier<B> for AuraVerifier<C, E> where
 		header: B::Header,
 		justification: Option<Justification>,
 		mut body: Option<Vec<B::Extrinsic>>,
-	) -> Result<(ImportBlock<B>, Option<Vec<Ed25519AuthorityId>>), String> {
+	) -> Result<(ImportBlock<B>, Option<Vec<AuthorityId>>), String> {
 		let mut inherent_data = self.inherent_data_providers.create_inherent_data().map_err(String::from)?;
 		let (timestamp_now, slot_now) = AuraSlotCompatible::extract_timestamp_and_slot(&inherent_data)
 			.map_err(|e| format!("Could not extract timestamp and slot: {:?}", e))?;
@@ -675,7 +679,7 @@ pub fn import_queue<B, C, E>(
 	B: Block,
 	C: 'static + Authorities<B> + ProvideRuntimeApi + Send + Sync,
 	C::Api: BlockBuilderApi<B>,
-	DigestItemFor<B>: CompatibleDigestItem + DigestItem<AuthorityId=Ed25519AuthorityId>,
+	DigestItemFor<B>: CompatibleDigestItem + DigestItem<AuthorityId=AuthorityId>,
 	E: 'static + ExtraVerification<B>,
 {
 	register_aura_inherent_data_provider(&inherent_data_providers, slot_duration.get())?;
@@ -696,7 +700,7 @@ mod tests {
 	use network::config::ProtocolConfig;
 	use parking_lot::Mutex;
 	use tokio::runtime::current_thread;
-	use keyring::Keyring;
+	use keyring::ed25519::Keyring;
 	use client::BlockchainEvents;
 	use test_client;
 
@@ -711,7 +715,7 @@ mod tests {
 		type Proposer = DummyProposer;
 		type Error = Error;
 
-		fn init(&self, parent_header: &<TestBlock as BlockT>::Header, _authorities: &[Ed25519AuthorityId])
+		fn init(&self, parent_header: &<TestBlock as BlockT>::Header, _authorities: &[AuthorityId])
 			-> Result<DummyProposer, Error>
 		{
 			Ok(DummyProposer(parent_header.number + 1, self.0.clone()))

substrate/core/consensus/common/src/error.rs

@@ -18,6 +18,7 @@
 use runtime_version::RuntimeVersion;
 use error_chain::{error_chain, error_chain_processing, impl_error_chain_processed,
 	impl_extract_backtrace, impl_error_chain_kind};
+use primitives::ed25519::{Public, Signature};
 
 error_chain! {
 	errors {
@@ -52,13 +53,13 @@ error_chain! {
 		}
 
 		/// Error checking signature
-		InvalidSignature(s: ::primitives::ed25519::Signature, a: ::primitives::Ed25519AuthorityId) {
+		InvalidSignature(s: Signature, a: Public) {
 			description("Message signature is invalid"),
 			display("Message signature {:?} by {:?} is invalid.", s, a),
 		}
 
 		/// Account is not an authority.
-		InvalidAuthority(a: ::primitives::Ed25519AuthorityId) {
+		InvalidAuthority(a: Public) {
 			description("Message sender is not a valid authority"),
 			display("Message sender {:?} is not a valid authority.", a),
 		}

substrate/core/consensus/common/src/offline_tracker.rs

@@ -112,25 +112,25 @@ impl<AuthorityId: Eq + Clone + std::hash::Hash> OfflineTracker<AuthorityId> {
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use primitives::Ed25519AuthorityId;
+	use primitives::ed25519::Public as AuthorityId;
 
 	#[test]
 	fn validator_offline() {
-		let mut tracker = OfflineTracker::<Ed25519AuthorityId>::new();
-		let v = [0; 32].into();
-		let v2 = [1; 32].into();
-		let v3 = [2; 32].into();
-		tracker.note_round_end(v, true);
-		tracker.note_round_end(v2, true);
-		tracker.note_round_end(v3, true);
+		let mut tracker = OfflineTracker::<AuthorityId>::new();
+		let v = AuthorityId::from_raw([0; 32]);
+		let v2 = AuthorityId::from_raw([1; 32]);
+		let v3 = AuthorityId::from_raw([2; 32]);
+		tracker.note_round_end(v.clone(), true);
+		tracker.note_round_end(v2.clone(), true);
+		tracker.note_round_end(v3.clone(), true);
 
 		let slash_time = REPORT_TIME + Duration::from_secs(5);
 		tracker.observed.get_mut(&v).unwrap().offline_since -= slash_time;
 		tracker.observed.get_mut(&v2).unwrap().offline_since -= slash_time;
 
-		assert_eq!(tracker.reports(&[v, v2, v3]), vec![0, 1]);
+		assert_eq!(tracker.reports(&[v.clone(), v2.clone(), v3.clone()]), vec![0, 1]);
 
-		tracker.note_new_block(&[v, v3]);
+		tracker.note_new_block(&[v.clone(), v3.clone()]);
 		assert_eq!(tracker.reports(&[v, v2, v3]), vec![0]);
 	}
 }

substrate/core/consensus/rhd/src/lib.rs

@@ -762,7 +762,7 @@ fn check_justification_signed_message<H>(
 		let auth_id = sig.signer.clone().into();
 		if !authorities.contains(&auth_id) { return None }
 
-		if ed25519::verify_strong(&sig.signature, message, &sig.signer) {
+		if ed25519::Pair::verify(&sig.signature, message, &sig.signer) {
 			Some(sig.signer.0)
 		} else {
 			None
@@ -838,7 +838,7 @@ pub fn check_vote<B: Block>(
 
 fn check_action<B: Block>(action: Action<B, B::Hash>, parent_hash: &B::Hash, sig: &LocalizedSignature) -> Result<(), Error> {
 	let message = localized_encode(*parent_hash, action);
-	if ed25519::verify_strong(&sig.signature, &message, &sig.signer) {
+	if ed25519::Pair::verify(&sig.signature, &message, &sig.signer) {
 		Ok(())
 	} else {
 		Err(CommonErrorKind::InvalidSignature(sig.signature.into(), sig.signer.clone().into()).into())
@@ -1315,7 +1315,7 @@ mod tests {
 
 	use runtime_primitives::testing::{Block as GenericTestBlock, Header as TestHeader};
 	use primitives::H256;
-	use self::keyring::Keyring;
+	use keyring::AuthorityKeyring;
 
 	type TestBlock = GenericTestBlock<()>;
 
@@ -1420,7 +1420,7 @@ mod tests {
 				start_round: 0,
 			})),
 			round_timeout_multiplier: 10,
-			key: Arc::new(Keyring::One.into()),
+			key: Arc::new(AuthorityKeyring::One.into()),
 			factory: DummyFactory
 		}
 	}
@@ -1446,10 +1446,10 @@ mod tests {
 	fn future_gets_preempted() {
 		let client = FakeClient {
 			authorities: vec![
-				Keyring::One.to_raw_public().into(),
-				Keyring::Two.to_raw_public().into(),
-				Keyring::Alice.to_raw_public().into(),
-				Keyring::Eve.to_raw_public().into(),
+				AuthorityKeyring::One.into(),
+				AuthorityKeyring::Two.into(),
+				AuthorityKeyring::Alice.into(),
+				AuthorityKeyring::Eve.into(),
 			],
 			imported_heights: Mutex::new(HashSet::new()),
 		};
@@ -1493,17 +1493,17 @@ mod tests {
 		let hash = [0xff; 32].into();
 
 		let authorities = vec![
-			Keyring::One.to_raw_public().into(),
-			Keyring::Two.to_raw_public().into(),
-			Keyring::Alice.to_raw_public().into(),
-			Keyring::Eve.to_raw_public().into(),
+			AuthorityKeyring::One.into(),
+			AuthorityKeyring::Two.into(),
+			AuthorityKeyring::Alice.into(),
+			AuthorityKeyring::Eve.into(),
 		];
 
 		let authorities_keys = vec![
-			Keyring::One.into(),
-			Keyring::Two.into(),
-			Keyring::Alice.into(),
-			Keyring::Eve.into(),
+			AuthorityKeyring::One.into(),
+			AuthorityKeyring::Two.into(),
+			AuthorityKeyring::Alice.into(),
+			AuthorityKeyring::Eve.into(),
 		];
 
 		let unchecked = UncheckedJustification(rhododendron::UncheckedJustification {
@@ -1554,8 +1554,8 @@ mod tests {
 		let parent_hash = Default::default();
 
 		let authorities = vec![
-			Keyring::Alice.to_raw_public().into(),
-			Keyring::Eve.to_raw_public().into(),
+			AuthorityKeyring::Alice.into(),
+			AuthorityKeyring::Eve.into(),
 		];
 
 		let block = TestBlock {
@@ -1563,7 +1563,7 @@ mod tests {
 			extrinsics: Default::default()
 		};
 
-		let proposal = sign_message(rhododendron::Message::Propose(1, block.clone()), &Keyring::Alice.pair(), parent_hash);;
+		let proposal = sign_message(rhododendron::Message::Propose(1, block.clone()), &AuthorityKeyring::Alice.pair(), parent_hash);;
 		if let rhododendron::LocalizedMessage::Propose(proposal) = proposal {
 			assert!(check_proposal(&authorities, &parent_hash, &proposal).is_ok());
 			let mut invalid_round = proposal.clone();
@@ -1577,7 +1577,7 @@ mod tests {
 		}
 
 		// Not an authority
-		let proposal = sign_message::<TestBlock>(rhododendron::Message::Propose(1, block), &Keyring::Bob.pair(), parent_hash);;
+		let proposal = sign_message::<TestBlock>(rhododendron::Message::Propose(1, block), &AuthorityKeyring::Bob.pair(), parent_hash);;
 		if let rhododendron::LocalizedMessage::Propose(proposal) = proposal {
 			assert!(check_proposal(&authorities, &parent_hash, &proposal).is_err());
 		} else {
@@ -1591,8 +1591,8 @@ mod tests {
 		let hash: H256 = [0xff; 32].into();
 
 		let authorities = vec![
-			Keyring::Alice.to_raw_public().into(),
-			Keyring::Eve.to_raw_public().into(),
+			AuthorityKeyring::Alice.into(),
+			AuthorityKeyring::Eve.into(),
 		];
 
 		let vote = sign_message::<TestBlock>(rhododendron::Message::Vote(rhododendron::Vote::Prepare(1, hash)), &Keyring::Alice.pair(), parent_hash);;
@@ -1618,10 +1618,10 @@ mod tests {
 	fn drop_bft_future_does_not_deadlock() {
 		let client = FakeClient {
 			authorities: vec![
-				Keyring::One.to_raw_public().into(),
-				Keyring::Two.to_raw_public().into(),
-				Keyring::Alice.to_raw_public().into(),
-				Keyring::Eve.to_raw_public().into(),
+				AuthorityKeyring::One.into(),
+				AuthorityKeyring::Two.into(),
+				AuthorityKeyring::Alice.into(),
+				AuthorityKeyring::Eve.into(),
 			],
 			imported_heights: Mutex::new(HashSet::new()),
 		};
@@ -1643,10 +1643,10 @@ mod tests {
 	fn bft_can_build_though_skipped() {
 		let client = FakeClient {
 			authorities: vec![
-				Keyring::One.to_raw_public().into(),
-				Keyring::Two.to_raw_public().into(),
-				Keyring::Alice.to_raw_public().into(),
-				Keyring::Eve.to_raw_public().into(),
+				AuthorityKeyring::One.into(),
+				AuthorityKeyring::Two.into(),
+				AuthorityKeyring::Alice.into(),
+				AuthorityKeyring::Eve.into(),
 			],
 			imported_heights: Mutex::new(HashSet::new()),
 		};

substrate/core/consensus/rhd/src/misbehaviour_check.rs

@@ -74,8 +74,7 @@ pub fn evaluate_misbehavior<B: Codec, H: Codec + Copy>(
 mod tests {
 	use super::*;
 
-	use keyring::ed25519;
-	use keyring::Keyring;
+	use keyring::AuthorityKeyring;
 	use rhododendron;
 
 	use runtime_primitives::testing::{H256, Block as RawBlock};
@@ -110,7 +109,7 @@ mod tests {
 
 	#[test]
 	fn evaluates_double_prepare() {
-		let key: ed25519::Pair = Keyring::One.into();
+		let key = AuthorityKeyring::One.pair();
 		let parent_hash = [0xff; 32].into();
 		let hash_1 = [0; 32].into();
 		let hash_2 = [1; 32].into();
@@ -127,7 +126,7 @@ mod tests {
 
 		// same signature twice is not misbehavior.
 		let signed = sign_prepare(&key, 1, hash_1, parent_hash);
-		assert!(evaluate_misbehavior::<Block, H256>(
+		assert!(!evaluate_misbehavior::<Block, H256>(
 			&key.public().into(),
 			parent_hash,
 			&MisbehaviorKind::BftDoublePrepare(
@@ -135,23 +134,23 @@ mod tests {
 				signed,
 				signed,
 			)
-		) == false);
+		));
 
 		// misbehavior has wrong target.
-		assert!(evaluate_misbehavior::<Block, H256>(
-			&Keyring::Two.to_raw_public().into(),
+		assert!(!evaluate_misbehavior::<Block, H256>(
+			&AuthorityKeyring::Two.into(),
 			parent_hash,
 			&MisbehaviorKind::BftDoublePrepare(
 				1,
 				sign_prepare(&key, 1, hash_1, parent_hash),
 				sign_prepare(&key, 1, hash_2, parent_hash),
 			)
-		) == false);
+		));
 	}
 
 	#[test]
 	fn evaluates_double_commit() {
-		let key: ed25519::Pair = Keyring::One.into();
+		let key = AuthorityKeyring::One.pair();
 		let parent_hash = [0xff; 32].into();
 		let hash_1 = [0; 32].into();
 		let hash_2 = [1; 32].into();
@@ -168,7 +167,7 @@ mod tests {
 
 		// same signature twice is not misbehavior.
 		let signed = sign_commit(&key, 1, hash_1, parent_hash);
-		assert!(evaluate_misbehavior::<Block, H256>(
+		assert!(!evaluate_misbehavior::<Block, H256>(
 			&key.public().into(),
 			parent_hash,
 			&MisbehaviorKind::BftDoubleCommit(
@@ -176,17 +175,17 @@ mod tests {
 				signed,
 				signed,
 			)
-		) == false);
+		));
 
 		// misbehavior has wrong target.
-		assert!(evaluate_misbehavior::<Block, H256>(
-			&Keyring::Two.to_raw_public().into(),
+		assert!(!evaluate_misbehavior::<Block, H256>(
+			&AuthorityKeyring::Two.into(),
 			parent_hash,
 			&MisbehaviorKind::BftDoubleCommit(
 				1,
 				sign_commit(&key, 1, hash_1, parent_hash),
 				sign_commit(&key, 1, hash_2, parent_hash),
 			)
-		) == false);
+		));
 	}
 }