Overhaul crypto (Schnorr/Ristretto, HDKD, BIP39) (#1795)

* Rijig to Ristretto

* Rebuild wasm

* adds compatibility test with the wasm module

* Add Ed25519-BIP39 support

* Bump subkey version

* Update CLI output

* New keys.

* Standard phrase/password/path keys.

* Subkey uses S-URI for secrets

* Move everything to use new HDKD crypto.

* Test fixes

* Ignore old test vector.

* fix the ^^ old test vector.

* Fix tests

* Test fixes

* Cleanups

* Fix broken key conversion logic in grandpa

CC @rphmeier

* Remove legacy Keyring usage

* Traitify `Pair`

* Replace Ed25519AuthorityId with ed25519::Public

* Expunge Ed25519AuthorityId type!

* Replace Sr25519AuthorityId with sr25519::Public

* Remove dodgy crypto type-punning conversions

* Fix some tests

* Avoid trait

* Deduplicate DeriveJunction string decode

* Remove cruft code

* Fix test

* Minor removals

* Build fix

* Subkey supports sign and verify

* Inspect works for public key URIs

* Remove more crypto type-punning

* Fix typo

* Fix tests

substrate/core/client/src/client.rs

@@ -1520,11 +1520,10 @@ impl<B, E, Block, RA> backend::AuxStore for Client<B, E, Block, RA>
 pub(crate) mod tests {
 	use std::collections::HashMap;
 	use super::*;
-	use keyring::Keyring;
 	use primitives::twox_128;
 	use runtime_primitives::traits::DigestItem as DigestItemT;
 	use runtime_primitives::generic::DigestItem;
-	use test_client::{self, TestClient};
+	use test_client::{self, TestClient, AccountKeyring, AuthorityKeyring};
 	use consensus::BlockOrigin;
 	use test_client::client::backend::Backend as TestBackend;
 	use test_client::BlockBuilderExt;
@@ -1541,10 +1540,10 @@ pub(crate) mod tests {
 	) {
 		// prepare block structure
 		let blocks_transfers = vec![
-			vec![(Keyring::Alice, Keyring::Dave), (Keyring::Bob, Keyring::Dave)],
-			vec![(Keyring::Charlie, Keyring::Eve)],
+			vec![(AccountKeyring::Alice, AccountKeyring::Dave), (AccountKeyring::Bob, AccountKeyring::Dave)],
+			vec![(AccountKeyring::Charlie, AccountKeyring::Eve)],
 			vec![],
-			vec![(Keyring::Alice, Keyring::Dave)],
+			vec![(AccountKeyring::Alice, AccountKeyring::Dave)],
 		];
 
 		// prepare client ang import blocks
@@ -1555,8 +1554,8 @@ pub(crate) mod tests {
 			let mut builder = remote_client.new_block().unwrap();
 			for (from, to) in block_transfers {
 				builder.push_transfer(Transfer {
-					from: from.to_raw_public().into(),
-					to: to.to_raw_public().into(),
+					from: from.into(),
+					to: to.into(),
 					amount: 1,
 					nonce: *nonces.entry(from).and_modify(|n| { *n = *n + 1 }).or_default(),
 				}).unwrap();
@@ -1571,12 +1570,12 @@ pub(crate) mod tests {
 		}
 
 		// prepare test cases
-		let alice = twox_128(&runtime::system::balance_of_key(Keyring::Alice.to_raw_public().into())).to_vec();
-		let bob = twox_128(&runtime::system::balance_of_key(Keyring::Bob.to_raw_public().into())).to_vec();
-		let charlie = twox_128(&runtime::system::balance_of_key(Keyring::Charlie.to_raw_public().into())).to_vec();
-		let dave = twox_128(&runtime::system::balance_of_key(Keyring::Dave.to_raw_public().into())).to_vec();
-		let eve = twox_128(&runtime::system::balance_of_key(Keyring::Eve.to_raw_public().into())).to_vec();
-		let ferdie = twox_128(&runtime::system::balance_of_key(Keyring::Ferdie.to_raw_public().into())).to_vec();
+		let alice = twox_128(&runtime::system::balance_of_key(AccountKeyring::Alice.into())).to_vec();
+		let bob = twox_128(&runtime::system::balance_of_key(AccountKeyring::Bob.into())).to_vec();
+		let charlie = twox_128(&runtime::system::balance_of_key(AccountKeyring::Charlie.into())).to_vec();
+		let dave = twox_128(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec();
+		let eve = twox_128(&runtime::system::balance_of_key(AccountKeyring::Eve.into())).to_vec();
+		let ferdie = twox_128(&runtime::system::balance_of_key(AccountKeyring::Ferdie.into())).to_vec();
 		let test_cases = vec![
 			(1, 4, alice.clone(), vec![(4, 0), (1, 0)]),
 			(1, 3, alice.clone(), vec![(1, 0)]),
@@ -1610,14 +1609,14 @@ pub(crate) mod tests {
 		assert_eq!(
 			client.runtime_api().balance_of(
 				&BlockId::Number(client.info().unwrap().chain.best_number),
-				Keyring::Alice.to_raw_public().into()
+				AccountKeyring::Alice.into()
 			).unwrap(),
 			1000
 		);
 		assert_eq!(
 			client.runtime_api().balance_of(
 				&BlockId::Number(client.info().unwrap().chain.best_number),
-				Keyring::Ferdie.to_raw_public().into()
+				AccountKeyring::Ferdie.into()
 			).unwrap(),
 			0
 		);
@@ -1629,9 +1628,9 @@ pub(crate) mod tests {
 
 		assert_eq!(client.info().unwrap().chain.best_number, 0);
 		assert_eq!(client.authorities_at(&BlockId::Number(0)).unwrap(), vec![
-			Keyring::Alice.to_raw_public().into(),
-			Keyring::Bob.to_raw_public().into(),
-			Keyring::Charlie.to_raw_public().into()
+			AuthorityKeyring::Alice.into(),
+			AuthorityKeyring::Bob.into(),
+			AuthorityKeyring::Charlie.into()
 		]);
 	}
 
@@ -1653,8 +1652,8 @@ pub(crate) mod tests {
 		let mut builder = client.new_block().unwrap();
 
 		builder.push_transfer(Transfer {
-			from: Keyring::Alice.to_raw_public().into(),
-			to: Keyring::Ferdie.to_raw_public().into(),
+			from: AccountKeyring::Alice.into(),
+			to: AccountKeyring::Ferdie.into(),
 			amount: 42,
 			nonce: 0,
 		}).unwrap();
@@ -1666,14 +1665,14 @@ pub(crate) mod tests {
 		assert_eq!(
 			client.runtime_api().balance_of(
 				&BlockId::Number(client.info().unwrap().chain.best_number),
-				Keyring::Alice.to_raw_public().into()
+				AccountKeyring::Alice.into()
 			).unwrap(),
 			958
 		);
 		assert_eq!(
 			client.runtime_api().balance_of(
 				&BlockId::Number(client.info().unwrap().chain.best_number),
-				Keyring::Ferdie.to_raw_public().into()
+				AccountKeyring::Ferdie.into()
 			).unwrap(),
 			42
 		);
@@ -1695,15 +1694,15 @@ pub(crate) mod tests {
 		let mut builder = client.new_block().unwrap();
 
 		builder.push_transfer(Transfer {
-			from: Keyring::Alice.to_raw_public().into(),
-			to: Keyring::Ferdie.to_raw_public().into(),
+			from: AccountKeyring::Alice.into(),
+			to: AccountKeyring::Ferdie.into(),
 			amount: 42,
 			nonce: 0,
 		}).unwrap();
 
 		assert!(builder.push_transfer(Transfer {
-			from: Keyring::Eve.to_raw_public().into(),
-			to: Keyring::Alice.to_raw_public().into(),
+			from: AccountKeyring::Eve.into(),
+			to: AccountKeyring::Alice.into(),
 			amount: 42,
 			nonce: 0,
 		}).is_err());
@@ -1789,8 +1788,8 @@ pub(crate) mod tests {
 		let mut builder = client.new_block_at(&BlockId::Hash(a1.hash())).unwrap();
 		// this push is required as otherwise B2 has the same hash as A2 and won't get imported
 		builder.push_transfer(Transfer {
-			from: Keyring::Alice.to_raw_public().into(),
-			to: Keyring::Ferdie.to_raw_public().into(),
+			from: AccountKeyring::Alice.into(),
+			to: AccountKeyring::Ferdie.into(),
 			amount: 41,
 			nonce: 0,
 		}).unwrap();
@@ -1809,8 +1808,8 @@ pub(crate) mod tests {
 		let mut builder = client.new_block_at(&BlockId::Hash(b2.hash())).unwrap();
 		// this push is required as otherwise C3 has the same hash as B3 and won't get imported
 		builder.push_transfer(Transfer {
-			from: Keyring::Alice.to_raw_public().into(),
-			to: Keyring::Ferdie.to_raw_public().into(),
+			from: AccountKeyring::Alice.into(),
+			to: AccountKeyring::Ferdie.into(),
 			amount: 1,
 			nonce: 1,
 		}).unwrap();
@@ -1821,8 +1820,8 @@ pub(crate) mod tests {
 		let mut builder = client.new_block_at(&BlockId::Hash(a1.hash())).unwrap();
 		// this push is required as otherwise D2 has the same hash as B2 and won't get imported
 		builder.push_transfer(Transfer {
-			from: Keyring::Alice.to_raw_public().into(),
-			to: Keyring::Ferdie.to_raw_public().into(),
+			from: AccountKeyring::Alice.into(),
+			to: AccountKeyring::Ferdie.into(),
 			amount: 1,
 			nonce: 0,
 		}).unwrap();
@@ -1910,8 +1909,8 @@ pub(crate) mod tests {
 		let mut builder = client.new_block_at(&BlockId::Hash(a1.hash())).unwrap();
 		// this push is required as otherwise B2 has the same hash as A2 and won't get imported
 		builder.push_transfer(Transfer {
-			from: Keyring::Alice.to_raw_public().into(),
-			to: Keyring::Ferdie.to_raw_public().into(),
+			from: AccountKeyring::Alice.into(),
+			to: AccountKeyring::Ferdie.into(),
 			amount: 41,
 			nonce: 0,
 		}).unwrap();
@@ -1930,8 +1929,8 @@ pub(crate) mod tests {
 		let mut builder = client.new_block_at(&BlockId::Hash(b2.hash())).unwrap();
 		// this push is required as otherwise C3 has the same hash as B3 and won't get imported
 		builder.push_transfer(Transfer {
-			from: Keyring::Alice.to_raw_public().into(),
-			to: Keyring::Ferdie.to_raw_public().into(),
+			from: AccountKeyring::Alice.into(),
+			to: AccountKeyring::Ferdie.into(),
 			amount: 1,
 			nonce: 1,
 		}).unwrap();
@@ -1942,8 +1941,8 @@ pub(crate) mod tests {
 		let mut builder = client.new_block_at(&BlockId::Hash(a1.hash())).unwrap();
 		// this push is required as otherwise D2 has the same hash as B2 and won't get imported
 		builder.push_transfer(Transfer {
-			from: Keyring::Alice.to_raw_public().into(),
-			to: Keyring::Ferdie.to_raw_public().into(),
+			from: AccountKeyring::Alice.into(),
+			to: AccountKeyring::Ferdie.into(),
 			amount: 1,
 			nonce: 0,
 		}).unwrap();

substrate/core/client/src/genesis.rs

@@ -41,14 +41,16 @@ pub fn construct_genesis_block<
 mod tests {
 	use super::*;
 	use parity_codec::{Encode, Decode, Joiner};
-	use keyring::Keyring;
 	use executor::{NativeExecutionDispatch, native_executor_instance};
 	use state_machine::{self, OverlayedChanges, ExecutionStrategy, InMemoryChangesTrieStorage};
 	use state_machine::backend::InMemory;
-	use test_client::runtime::genesismap::{GenesisConfig, additional_storage_with_genesis};
-	use test_client::runtime::{Hash, Transfer, Block, BlockNumber, Header, Digest, Extrinsic};
+	use test_client::{
+		runtime::genesismap::{GenesisConfig, additional_storage_with_genesis},
+		runtime::{Hash, Transfer, Block, BlockNumber, Header, Digest, Extrinsic},
+		AccountKeyring, AuthorityKeyring
+	};
 	use runtime_primitives::traits::BlakeTwo256;
-	use primitives::{Blake2Hasher, ed25519::{Public, Pair}};
+	use primitives::Blake2Hasher;
 	use hex::*;
 
 	native_executor_instance!(Executor, test_client::runtime::api::dispatch, test_client::runtime::native_version, include_bytes!("../../test-runtime/wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm"));
@@ -67,7 +69,7 @@ mod tests {
 		use trie::ordered_trie_root;
 
 		let transactions = txs.into_iter().map(|tx| {
-			let signature = Pair::from(Keyring::from_public(Public::from_raw(tx.from.to_fixed_bytes())).unwrap())
+			let signature = AccountKeyring::from_public(&tx.from).unwrap()
 				.sign(&tx.encode()).into();
 
 			Extrinsic::Transfer(tx, signature)
@@ -75,7 +77,6 @@ mod tests {
 
 		let extrinsics_root = ordered_trie_root::<Blake2Hasher, _, _>(transactions.iter().map(Encode::encode)).into();
 
-		println!("root before: {:?}", extrinsics_root);
 		let mut header = Header {
 			parent_hash,
 			number,
@@ -121,7 +122,6 @@ mod tests {
 			ExecutionStrategy::NativeElseWasm,
 		).unwrap();
 		header = Header::decode(&mut &ret_data[..]).unwrap();
-		println!("root after: {:?}", header.extrinsics_root);
 
 		(vec![].and(&Block { header, extrinsics: transactions }), hash)
 	}
@@ -133,8 +133,8 @@ mod tests {
 			genesis_hash,
 			hex!("25e5b37074063ab75c889326246640729b40d0c86932edc527bc80db0e04fe5c").into(),
 			vec![Transfer {
-				from: Keyring::One.to_raw_public().into(),
-				to: Keyring::Two.to_raw_public().into(),
+				from: AccountKeyring::One.into(),
+				to: AccountKeyring::Two.into(),
 				amount: 69,
 				nonce: 0,
 			}]
@@ -143,8 +143,10 @@ mod tests {
 
 	#[test]
 	fn construct_genesis_should_work_with_native() {
-		let mut storage = GenesisConfig::new_simple(
-			vec![Keyring::One.to_raw_public().into(), Keyring::Two.to_raw_public().into()], 1000
+		let mut storage = GenesisConfig::new(false,
+			vec![AuthorityKeyring::One.into(), AuthorityKeyring::Two.into()],
+			vec![AccountKeyring::One.into(), AccountKeyring::Two.into()],
+			1000
 		).genesis_map();
 		let state_root = BlakeTwo256::trie_root(storage.clone().into_iter());
 		let block = construct_genesis_block::<Block>(state_root);
@@ -169,8 +171,10 @@ mod tests {
 
 	#[test]
 	fn construct_genesis_should_work_with_wasm() {
-		let mut storage = GenesisConfig::new_simple(
-			vec![Keyring::One.to_raw_public().into(), Keyring::Two.to_raw_public().into()], 1000
+		let mut storage = GenesisConfig::new(false,
+			vec![AuthorityKeyring::One.into(), AuthorityKeyring::Two.into()],
+			vec![AccountKeyring::One.into(), AccountKeyring::Two.into()],
+			1000
 		).genesis_map();
 		let state_root = BlakeTwo256::trie_root(storage.clone().into_iter());
 		let block = construct_genesis_block::<Block>(state_root);
@@ -194,10 +198,11 @@ mod tests {
 	}
 
 	#[test]
-	#[should_panic]
 	fn construct_genesis_with_bad_transaction_should_panic() {
-		let mut storage = GenesisConfig::new_simple(
-			vec![Keyring::One.to_raw_public().into(), Keyring::Two.to_raw_public().into()], 68
+		let mut storage = GenesisConfig::new(false,
+			vec![AuthorityKeyring::One.into(), AuthorityKeyring::Two.into()],
+			vec![AccountKeyring::One.into(), AccountKeyring::Two.into()],
+			68
 		).genesis_map();
 		let state_root = BlakeTwo256::trie_root(storage.clone().into_iter());
 		let block = construct_genesis_block::<Block>(state_root);
@@ -208,7 +213,7 @@ mod tests {
 		let (b1data, _b1hash) = block1(genesis_hash, &backend);
 
 		let mut overlay = OverlayedChanges::default();
-		let _ = state_machine::new(
+		let r = state_machine::new(
 			&backend,
 			Some(&InMemoryChangesTrieStorage::new()),
 			&mut overlay,
@@ -217,6 +222,7 @@ mod tests {
 			&b1data,
 		).execute(
 			ExecutionStrategy::NativeElseWasm,
-		).unwrap();
+		);
+		assert!(r.is_err());
 	}
 }

substrate/core/client/src/light/backend.rs

@@ -35,7 +35,7 @@ use hash_db::Hasher;
 use trie::MemoryDB;
 use heapsize::HeapSizeOf;
 
-const IN_MEMORY_EXPECT_PROOF: &'static str = "InMemory state backend has Void error type and always suceeds; qed";
+const IN_MEMORY_EXPECT_PROOF: &str = "InMemory state backend has Void error type and always suceeds; qed";
 
 /// Light client backend.
 pub struct Backend<S, F, H> {

substrate/core/client/src/light/fetcher.rs

@@ -390,12 +390,13 @@ impl<'a, H, Number, Hash> ChangesTrieRootsStorage<H> for RootsStorage<'a, Number
 pub mod tests {
 	use futures::future::{ok, err, FutureResult};
 	use parking_lot::Mutex;
-	use keyring::Keyring;
 	use crate::client::tests::prepare_client_with_key_changes;
 	use executor::{self, NativeExecutionDispatch};
 	use crate::error::Error as ClientError;
-	use test_client::{self, TestClient, blockchain::HeaderBackend};
-	use test_client::runtime::{self, Hash, Block, Header};
+	use test_client::{
+		self, TestClient, blockchain::HeaderBackend, AccountKeyring,
+		runtime::{self, Hash, Block, Header}
+	};
 	use consensus::BlockOrigin;
 
 	use crate::in_mem::{Blockchain as InMemoryBlockchain};
@@ -583,7 +584,7 @@ pub mod tests {
 		// we're testing this test case here:
 		// (1, 4, dave.clone(), vec![(4, 0), (1, 1), (1, 0)]),
 		let (remote_client, remote_roots, _) = prepare_client_with_key_changes();
-		let dave = twox_128(&runtime::system::balance_of_key(Keyring::Dave.to_raw_public().into())).to_vec();
+		let dave = twox_128(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec();
 		let dave = StorageKey(dave);
 
 		// 'fetch' changes proof from remote node:
@@ -695,7 +696,7 @@ pub mod tests {
 		let (remote_client, remote_roots, _) = prepare_client_with_key_changes();
 		let local_cht_root = cht::compute_root::<Header, Blake2Hasher, _>(
 			4, 0, remote_roots.iter().cloned().map(|ct| Ok(Some(ct)))).unwrap();
-		let dave = twox_128(&runtime::system::balance_of_key(Keyring::Dave.to_raw_public().into())).to_vec();
+		let dave = twox_128(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec();
 		let dave = StorageKey(dave);
 
 		// 'fetch' changes proof from remote node: