Overhaul crypto (Schnorr/Ristretto, HDKD, BIP39) (#1795)

* Rijig to Ristretto

* Rebuild wasm

* adds compatibility test with the wasm module

* Add Ed25519-BIP39 support

* Bump subkey version

* Update CLI output

* New keys.

* Standard phrase/password/path keys.

* Subkey uses S-URI for secrets

* Move everything to use new HDKD crypto.

* Test fixes

* Ignore old test vector.

* fix the ^^ old test vector.

* Fix tests

* Test fixes

* Cleanups

* Fix broken key conversion logic in grandpa

CC @rphmeier

* Remove legacy Keyring usage

* Traitify `Pair`

* Replace Ed25519AuthorityId with ed25519::Public

* Expunge Ed25519AuthorityId type!

* Replace Sr25519AuthorityId with sr25519::Public

* Remove dodgy crypto type-punning conversions

* Fix some tests

* Avoid trait

* Deduplicate DeriveJunction string decode

* Remove cruft code

* Fix test

* Minor removals

* Build fix

* Subkey supports sign and verify

* Inspect works for public key URIs

* Remove more crypto type-punning

* Fix typo

* Fix tests

substrate/subkey/src/cli.yml

@@ -1,12 +1,7 @@
 name: subkey
 author: "Parity Team <admin@parity.io>"
-about: A substrate key utility
+about: Utility for generating and restoring with Substrate keys
 args:
-  - ed25519-legacy:
-      short: o
-      long: legacy
-      help: Use legacy, outdated Ed25519 cryptography
-      takes_value: false
   - ed25519:
       short: e
       long: ed25519
@@ -26,15 +21,41 @@ args:
 subcommands:
   - generate:
       about: Generate a random account
-  - restore:
-      about: Gets a public key and a SS58 address from the provided seed phrase
+  - inspect:
+      about: Gets a public key and a SS58 address from the provided Secret URI
       args:
-        - seed:
+        - uri:
             index: 1
             required: true
-            help: 32 bytes long seed phrase used to restore the public key. If the provided seed is shorter than that, then
-                  it will be right-padded with 0x20 bytes (ASCII space). If the provided seed is longer than
-                  32 bytes then seed will be truncated.
+            help: A Key URI to be inspected. May be a secret seed, secret URI (with derivation paths and password), SS58 or public URI.
+  - sign:
+      about: Sign a message, provided on STDIN, with a given (secret) key
+      args:
+        - suri:
+            index: 1
+            required: true
+            help: The secret key URI.
+        - hex:
+            short: h
+            long: hex
+            help: The message on STDIN is hex-encoded data
+            takes_value: false
+  - verify:
+      about: Verify a signature for a message, provided on STDIN, with a given (public or secret) key
+      args:
+        - sig:
+            index: 1
+            required: true
+            help: Signature, hex-encoded.
+        - uri:
+            index: 2
+            required: true
+            help: The public or secret key URI.
+        - hex:
+            short: h
+            long: hex
+            help: The message on STDIN is hex-encoded data
+            takes_value: false
   - vanity:
       about: Generate a seed that provides a vanity address
       args:
@@ -47,9 +68,3 @@ subcommands:
             help: Number of keys to generate
             takes_value: true
             default_value: "1"
-  - query:
-      about: Query an account by its seed
-      args:
-        - seed:
-            index: 1
-            help: The 0x prefixed seed

substrate/subkey/src/main.rs

@@ -21,30 +21,35 @@ extern crate test;
 extern crate substrate_bip39;
 extern crate rustc_hex;
 
+use std::io::{stdin, Read};
 use clap::load_yaml;
 use rand::{RngCore, rngs::OsRng};
-use substrate_bip39::{mini_secret_from_entropy};
+use substrate_bip39::mini_secret_from_entropy;
 use bip39::{Mnemonic, Language, MnemonicType};
-use substrate_primitives::{ed25519, sr25519, hexdisplay::HexDisplay};
+use substrate_primitives::{ed25519, sr25519, hexdisplay::HexDisplay, Pair, crypto::Ss58Codec};
 use schnorrkel::keys::MiniSecretKey;
-use rustc_hex::FromHex;
 
 mod vanity;
 
 trait Crypto {
 	type Seed: AsRef<[u8]> + AsMut<[u8]> + Sized + Default;
-	type Pair;
+	type Pair: Pair;
 	fn generate_phrase() -> String {
 		Mnemonic::new(MnemonicType::Words12, Language::English).phrase().to_owned()
 	}
-	fn generate_seed() -> Self::Seed;
+	fn generate_seed() -> Self::Seed {
+		let mut seed: Self::Seed = Default::default();
+		OsRng::new().unwrap().fill_bytes(seed.as_mut());
+		seed
+	}
 	fn seed_from_phrase(phrase: &str, password: Option<&str>) -> Self::Seed;
 	fn pair_from_seed(seed: &Self::Seed) -> Self::Pair;
-	fn pair_from_phrase(phrase: &str, password: Option<&str>) -> Self::Pair {
+	fn pair_from_suri(phrase: &str, password: Option<&str>) -> Self::Pair {
 		Self::pair_from_seed(&Self::seed_from_phrase(phrase, password))
 	}
 	fn ss58_from_pair(pair: &Self::Pair) -> String;
 	fn public_from_pair(pair: &Self::Pair) -> Vec<u8>;
+	fn seed_from_pair(_pair: &Self::Pair) -> Option<&Self::Seed> { None }
 	fn print_from_seed(seed: &Self::Seed) {
 		let pair = Self::pair_from_seed(seed);
 		println!("Seed 0x{} is account:\n  Public key (hex): 0x{}\n  Address (SS58): {}",
@@ -63,43 +68,43 @@ trait Crypto {
 			Self::ss58_from_pair(&pair)
 		);
 	}
+	fn print_from_uri(uri: &str, password: Option<&str>) where <Self::Pair as Pair>::Public: Sized + Ss58Codec + AsRef<[u8]> {
+		if let Ok(pair) = Self::Pair::from_string(uri, password) {
+			let seed_text = Self::seed_from_pair(&pair)
+				.map_or_else(Default::default, |s| format!("\n  Seed: 0x{}", HexDisplay::from(&s.as_ref())));
+			println!("Secret Key URI `{}` is account:{}\n  Public key (hex): 0x{}\n  Address (SS58): {}",
+				uri,
+				seed_text,
+				HexDisplay::from(&Self::public_from_pair(&pair)),
+				Self::ss58_from_pair(&pair)
+			);
+		}
+		if let Ok(public) = <Self::Pair as Pair>::Public::from_string(uri) {
+			println!("Public Key URI `{}` is account:\n  Public key (hex): 0x{}\n  Address (SS58): {}",
+				uri,
+				HexDisplay::from(&public.as_ref()),
+				public.to_ss58check()
+			);
+		}
+	}
 }
 
-struct OriginalEd25519;
+struct Ed25519;
 
-impl Crypto for OriginalEd25519 {
+impl Crypto for Ed25519 {
 	type Seed = [u8; 32];
 	type Pair = ed25519::Pair;
 
-	fn generate_seed() -> Self::Seed {
-		let mut seed = [0u8; 32];
-		OsRng::new().unwrap().fill_bytes(&mut seed[..]);
-		seed
-	}
-
 	fn seed_from_phrase(phrase: &str, password: Option<&str>) -> Self::Seed {
-		if password.is_some() {
-			panic!("Ed25519 original doesn't support passwords")
-		}
-
-		let mut raw_seed = phrase.as_bytes();
-
-		if raw_seed.len() > 32 {
-			raw_seed = &raw_seed[..32];
-			println!("seed is too long and will be truncated to: {}", HexDisplay::from(&raw_seed));
-		}
-
-		// Copy the raw_seed into a buffer that already contains ' ' 0x20.
-		// This will effectively get us padding for seeds shorter than 32.
-		let mut seed = [' ' as u8; 32];
-		let len = raw_seed.len().min(32);
-		seed[..len].copy_from_slice(&raw_seed[..len]);
-		seed
+		Sr25519::seed_from_phrase(phrase, password)
 	}
-
-	fn pair_from_seed(seed: &Self::Seed) -> Self::Pair { ed25519::Pair::from_seed(seed) }
+	fn pair_from_suri(suri: &str, password_override: Option<&str>) -> Self::Pair {
+		ed25519::Pair::from_legacy_string(suri, password_override)
+	}
+	fn pair_from_seed(seed: &Self::Seed) -> Self::Pair { ed25519::Pair::from_seed(seed.clone()) }
 	fn ss58_from_pair(pair: &Self::Pair) -> String { pair.public().to_ss58check() }
 	fn public_from_pair(pair: &Self::Pair) -> Vec<u8> { (&pair.public().0[..]).to_owned() }
+	fn seed_from_pair(pair: &Self::Pair) -> Option<&Self::Seed> { Some(pair.seed()) }
 }
 
 struct Sr25519;
@@ -108,12 +113,6 @@ impl Crypto for Sr25519 {
 	type Seed = [u8; 32];
 	type Pair = sr25519::Pair;
 
-	fn generate_seed() -> Self::Seed {
-		let mut seed = [0u8; 32];
-		OsRng::new().unwrap().fill_bytes(&mut seed[..]);
-		seed
-	}
-
 	fn seed_from_phrase(phrase: &str, password: Option<&str>) -> Self::Seed {
 		mini_secret_from_entropy(
 			Mnemonic::from_phrase(phrase, Language::English)
@@ -127,11 +126,8 @@ impl Crypto for Sr25519 {
 			.to_bytes()
 	}
 
-	fn pair_from_phrase(phrase: &str, password: Option<&str>) -> Self::Pair {
-		sr25519::Pair::from_phrase(phrase, password)
-			.unwrap_or_else(||
-				panic!("Phrase is not a valid BIP-39 phrase: \n    {}", phrase)
-			)
+	fn pair_from_suri(suri: &str, password: Option<&str>) -> Self::Pair {
+		sr25519::Pair::from_string(suri, password).expect("Invalid phrase")
 	}
 
 	fn pair_from_seed(seed: &Self::Seed) -> Self::Pair {
@@ -143,7 +139,10 @@ impl Crypto for Sr25519 {
 	fn public_from_pair(pair: &Self::Pair) -> Vec<u8> { (&pair.public().0[..]).to_owned() }
 }
 
-fn execute<C: Crypto<Seed=[u8; 32]>>(matches: clap::ArgMatches) {
+fn execute<C: Crypto<Seed=[u8; 32]>>(matches: clap::ArgMatches) where
+	<<C as Crypto>::Pair as Pair>::Signature: AsRef<[u8]> + AsMut<[u8]> + Default,
+	<<C as Crypto>::Pair as Pair>::Public: Sized + AsRef<[u8]> + Ss58Codec + AsRef<<<C as Crypto>::Pair as Pair>::Public>,
+{
 	let password = matches.value_of("password");
 	match matches.subcommand() {
 		("generate", Some(_matches)) => {
@@ -156,38 +155,61 @@ fn execute<C: Crypto<Seed=[u8; 32]>>(matches: clap::ArgMatches) {
 			let key = vanity::generate_key::<C>(&desired).expect("Key generation failed");
 			C::print_from_seed(&key.seed);
 		}
-		("restore", Some(matches)) => {
-			let phrase = matches.value_of("seed")
-				.expect("seed parameter is required; thus it can't be None; qed");
-			C::print_from_phrase(phrase, password);
+		("inspect", Some(matches)) => {
+			// TODO: Accept public key with derivation path.
+			let uri = matches.value_of("uri")
+				.expect("URI parameter is required; thus it can't be None; qed");
+			C::print_from_uri(uri, password);
 		},
-		("query", Some(matches)) => {
-			let seed_data = matches.value_of("seed")
-				.expect("seed parameter is required; thus it can't be None; qed");
-			let seed_data = if seed_data.starts_with("0x") {
-				&seed_data[2..]
-			} else {
-				seed_data
-			};
-			let seed_data: Vec<u8> = seed_data.from_hex().expect("seed is not valid hex");
-			let correct_size = ::std::mem::size_of::<C::Seed>();
-			if seed_data.len() != correct_size {
-				panic!("Seed is incorrect size. It must be {} bytes for this cryptography", correct_size);
+		("sign", Some(matches)) => {
+			let suri = matches.value_of("suri")
+				.expect("secret URI parameter is required; thus it can't be None; qed");
+			let pair = C::pair_from_suri(suri, password);
+			let mut message = vec![];
+			stdin().lock().read_to_end(&mut message).expect("Error reading from stdin");
+			if matches.is_present("hex") {
+				message = hex::decode(&message).expect("Invalid hex in message");
 			}
-			let mut seed = C::Seed::default();
-			seed.as_mut().copy_from_slice(&seed_data);
-			C::print_from_seed(&seed);
-		},
+			let sig = pair.sign(&message);
+			println!("{}", hex::encode(&sig));
+		}
+		("verify", Some(matches)) => {
+			let sig_data = matches.value_of("sig")
+				.expect("signature parameter is required; thus it can't be None; qed");
+			let mut sig = <<C as Crypto>::Pair as Pair>::Signature::default();
+			let sig_data = hex::decode(sig_data).expect("signature is invalid hex");
+			if sig_data.len() != sig.as_ref().len() {
+				panic!("signature is an invalid length. {} bytes is not the expected value of {} bytes", sig_data.len(), sig.as_ref().len());
+			}
+			sig.as_mut().copy_from_slice(&sig_data);
+			let uri = matches.value_of("uri")
+				.expect("public uri parameter is required; thus it can't be None; qed");
+			let pubkey = <<C as Crypto>::Pair as Pair>::Public::from_string(uri).ok().or_else(||
+				<C as Crypto>::Pair::from_string(uri, password).ok().map(|p| p.public())
+			).expect("Invalid URI; expecting either a secret URI or a public URI.");
+			let mut message = vec![];
+			stdin().lock().read_to_end(&mut message).expect("Error reading from stdin");
+			if matches.is_present("hex") {
+				message = hex::decode(&message).expect("Invalid hex in message");
+			}
+			if <<C as Crypto>::Pair as Pair>::verify(&sig, &message, &pubkey) {
+				println!("Signature verifies correctly.")
+			} else {
+				println!("Signature invalid.")
+			}
+		}
 		_ => print_usage(&matches),
 	}
 }
 
 fn main() {
 	let yaml = load_yaml!("cli.yml");
-	let matches = clap::App::from_yaml(yaml).get_matches();
+	let matches = clap::App::from_yaml(yaml)
+		.version(env!("CARGO_PKG_VERSION"))
+		.get_matches();
 
-	if matches.is_present("ed25519original") {
-		execute::<OriginalEd25519>(matches)
+	if matches.is_present("ed25519") {
+		execute::<Ed25519>(matches)
 	} else {
 		execute::<Sr25519>(matches)
 	}

substrate/subkey/src/vanity.rs

@@ -104,13 +104,14 @@ pub(super) fn generate_key<C: Crypto<Seed=[u8; 32]>>(desired: &str) -> Result<Ke
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use super::super::OriginalEd25519;
+	use super::super::Ed25519;
+	use substrate_primitives::Pair;
 	#[cfg(feature = "bench")]
 	use test::Bencher;
 
 	#[test]
 	fn test_generation_with_single_char() {
-		assert!(generate_key::<OriginalEd25519>("j").unwrap().pair.public().to_ss58check().contains("j"));
+		assert!(generate_key::<Ed25519>("j").unwrap().pair.public().to_ss58check().contains("j"));
 	}
 
 	#[test]