Refactor key management (#3296)

* Add Call type to extensible transactions.

Cleanup some naming

* Merge Resource and BlockExhausted into just Exhausted

* Fix

* Another fix

* Call

* Some fixes

* Fix srml tests.

* Fix all tests.

* Refactor crypto so each application of it has its own type.

* Introduce new AuthorityProvider API into Aura

This will eventually allow for dynamic determination of authority
keys and avoid having to set them directly on CLI.

* Introduce authority determinator for Babe.

Experiment with modular consensus API.

* Work in progress to introduce KeyTypeId and avoid polluting API
with validator IDs

* Finish up drafting imonline

* Rework offchain workers API.

* Rework API implementation.

* Make it compile for wasm, simplify app_crypto.

* Fix compilation of im-online.

* Fix compilation of im-online.

* Fix more compilation errors.

* Make it compile.

* Fixing tests.

* Rewrite `keystore`

* Fix session tests

* Bring back `TryFrom`'s'

* Fix `srml-grandpa`

* Fix `srml-aura`

* Fix consensus babe

* More fixes

* Make service generate keys from dev_seed

* Build fixes

* Remove offchain tests

* More fixes and cleanups

* Fixes finality grandpa

* Fix `consensus-aura`

* Fix cli

* Fix `node-cli`

* Fix chain_spec builder

* Fix doc tests

* Add authority getter for grandpa.

* Test fix

* Fixes

* Make keystore accessible from the runtime

* Move app crypto to its own crate

* Update `Cargo.lock`

* Make the crypto stuff usable from the runtime

* Adds some runtime crypto tests

* Use last finalized block for grandpa authority

* Fix warning

* Adds `SessionKeys` runtime api

* Remove `FinalityPair` and `ConsensusPair`

* Minor governance tweaks to get it inline with docs.

* Make the governance be up to date with the docs.

* Build fixes.

* Generate the inital session keys

* Failing keystore is a hard error

* Make babe work again

* Fix grandpa

* Fix tests

* Disable `keystore` in consensus critical stuff

* Build fix.

* ImOnline supports multiple authorities at once.

* Update core/application-crypto/src/ed25519.rs

* Merge branch 'master' into gav-in-progress

* Remove unneeded code for now.

* Some `session` testing

* Support querying the public keys

* Cleanup offchain

* Remove warnings

* More cleanup

* Apply suggestions from code review

Co-Authored-By: Benjamin Kampmann <ben.kampmann@googlemail.com>

* More cleanups

* JSONRPC API for setting keys.

Also, rename traits::KeyStore* -> traits::BareCryptoStore*

* Bad merge

* Fix integration tests

* Fix test build

* Test fix

* Fixes

* Warnings

* Another warning

* Bump version.

substrate/core/rpc/src/author/error.rs

@@ -37,6 +37,18 @@ pub enum Error {
 	/// Incorrect extrinsic format.
 	#[display(fmt="Invalid extrinsic format: {}", _0)]
 	BadFormat(codec::Error),
+	/// Incorrect seed phrase.
+	#[display(fmt="Invalid seed phrase/SURI")]
+	BadSeedPhrase,
+	/// Key type ID has an unknown format.
+	#[display(fmt="Invalid key type ID format (should be of length four)")]
+	BadKeyType,
+	/// Key type ID has some unsupported crypto.
+	#[display(fmt="The crypto of key type ID is unknown")]
+	UnsupportedKeyType,
+	/// Some random issue with the key store. Shouldn't happen.
+	#[display(fmt="The key store is unavailable")]
+	KeyStoreUnavailable,
 }
 
 impl std::error::Error for Error {

substrate/core/rpc/src/author/mod.rs

@@ -22,7 +22,7 @@ pub mod hash;
 #[cfg(test)]
 mod tests;
 
-use std::sync::Arc;
+use std::{sync::Arc, convert::TryInto};
 
 use client::{self, Client};
 use crate::rpc::futures::{Sink, Stream, Future};
@@ -31,9 +31,12 @@ use jsonrpc_derive::rpc;
 use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId};
 use log::warn;
 use codec::{Encode, Decode};
-use primitives::{Bytes, Blake2Hasher, H256};
+use primitives::{
+	Bytes, Blake2Hasher, H256, ed25519, sr25519, crypto::{Pair, Public, key_types},
+	traits::BareCryptoStorePtr
+};
 use sr_primitives::{generic, traits};
-use self::error::Result;
+use self::error::{Error, Result};
 use transaction_pool::{
 	txpool::{
 		ChainApi as PoolChainApi,
@@ -57,21 +60,46 @@ pub trait AuthorApi<Hash, BlockHash> {
 	#[rpc(name = "author_submitExtrinsic")]
 	fn submit_extrinsic(&self, extrinsic: Bytes) -> Result<Hash>;
 
+	/// Insert a key into the keystore.
+	#[rpc(name = "author_insertKey")]
+	fn insert_key(&self,
+		key_type: String,
+		suri: String,
+		maybe_public: Option<Bytes>
+	) -> Result<Bytes>;
+
 	/// Returns all pending extrinsics, potentially grouped by sender.
 	#[rpc(name = "author_pendingExtrinsics")]
 	fn pending_extrinsics(&self) -> Result<Vec<Bytes>>;
 
 	/// Remove given extrinsic from the pool and temporarily ban it to prevent reimporting.
 	#[rpc(name = "author_removeExtrinsic")]
-	fn remove_extrinsic(&self, bytes_or_hash: Vec<hash::ExtrinsicOrHash<Hash>>) -> Result<Vec<Hash>>;
+	fn remove_extrinsic(&self,
+		bytes_or_hash: Vec<hash::ExtrinsicOrHash<Hash>>
+	) -> Result<Vec<Hash>>;
 
 	/// Submit an extrinsic to watch.
-	#[pubsub(subscription = "author_extrinsicUpdate", subscribe, name = "author_submitAndWatchExtrinsic")]
-	fn watch_extrinsic(&self, metadata: Self::Metadata, subscriber: Subscriber<Status<Hash, BlockHash>>, bytes: Bytes);
+	#[pubsub(
+		subscription = "author_extrinsicUpdate",
+		subscribe,
+		name = "author_submitAndWatchExtrinsic"
+	)]
+	fn watch_extrinsic(&self,
+		metadata: Self::Metadata,
+		subscriber: Subscriber<Status<Hash, BlockHash>>,
+		bytes: Bytes
+	);
 
 	/// Unsubscribe from extrinsic watching.
-	#[pubsub(subscription = "author_extrinsicUpdate", unsubscribe, name = "author_unwatchExtrinsic")]
-	fn unwatch_extrinsic(&self, metadata: Option<Self::Metadata>, id: SubscriptionId) -> Result<bool>;
+	#[pubsub(
+		subscription = "author_extrinsicUpdate",
+		unsubscribe,
+		name = "author_unwatchExtrinsic"
+	)]
+	fn unwatch_extrinsic(&self,
+		metadata: Option<Self::Metadata>,
+		id: SubscriptionId
+	) -> Result<bool>;
 }
 
 /// Authoring API
@@ -82,6 +110,8 @@ pub struct Author<B, E, P, RA> where P: PoolChainApi + Sync + Send + 'static {
 	pool: Arc<Pool<P>>,
 	/// Subscriptions manager
 	subscriptions: Subscriptions,
+	/// The key store.
+	keystore: BareCryptoStorePtr,
 }
 
 impl<B, E, P, RA> Author<B, E, P, RA> where P: PoolChainApi + Sync + Send + 'static {
@@ -90,11 +120,13 @@ impl<B, E, P, RA> Author<B, E, P, RA> where P: PoolChainApi + Sync + Send + 'sta
 		client: Arc<Client<B, E, <P as PoolChainApi>::Block, RA>>,
 		pool: Arc<Pool<P>>,
 		subscriptions: Subscriptions,
+		keystore: BareCryptoStorePtr,
 	) -> Self {
 		Author {
 			client,
 			pool,
 			subscriptions,
+			keystore,
 		}
 	}
 }
@@ -105,10 +137,38 @@ impl<B, E, P, RA> AuthorApi<ExHash<P>, BlockHash<P>> for Author<B, E, P, RA> whe
 	P: PoolChainApi + Sync + Send + 'static,
 	P::Block: traits::Block<Hash=H256>,
 	P::Error: 'static,
-	RA: Send + Sync + 'static
+	RA: Send + Sync + 'static,
 {
 	type Metadata = crate::metadata::Metadata;
 
+	fn insert_key(&self,
+		key_type: String,
+		suri: String,
+		maybe_public: Option<Bytes>,
+	) -> Result<Bytes> {
+		let key_type = key_type.as_str().try_into().map_err(|_| Error::BadKeyType)?;
+		let mut keystore = self.keystore.write();
+		let maybe_password = keystore.password();
+		let public = match maybe_public {
+			Some(public) => public.0,
+			None => {
+				let maybe_public = match key_type {
+					key_types::BABE | key_types::IM_ONLINE | key_types::SR25519 =>
+						sr25519::Pair::from_string(&suri, maybe_password)
+							.map(|pair| pair.public().to_raw_vec()),
+					key_types::GRANDPA | key_types::ED25519 =>
+						ed25519::Pair::from_string(&suri, maybe_password)
+							.map(|pair| pair.public().to_raw_vec()),
+					_ => Err(Error::UnsupportedKeyType)?,
+				};
+				maybe_public.map_err(|_| Error::BadSeedPhrase)?
+			}
+		};
+		keystore.insert_unknown(key_type, &suri, &public[..])
+			.map_err(|_| Error::KeyStoreUnavailable)?;
+		Ok(public.into())
+	}
+
 	fn submit_extrinsic(&self, ext: Bytes) -> Result<ExHash<P>> {
 		let xt = Decode::decode(&mut &ext[..])?;
 		let best_block_hash = self.client.info().chain.best_hash;
@@ -124,7 +184,9 @@ impl<B, E, P, RA> AuthorApi<ExHash<P>, BlockHash<P>> for Author<B, E, P, RA> whe
 		Ok(self.pool.ready().map(|tx| tx.data.encode().into()).collect())
 	}
 
-	fn remove_extrinsic(&self, bytes_or_hash: Vec<hash::ExtrinsicOrHash<ExHash<P>>>) -> Result<Vec<ExHash<P>>> {
+	fn remove_extrinsic(&self,
+		bytes_or_hash: Vec<hash::ExtrinsicOrHash<ExHash<P>>>
+	) -> Result<Vec<ExHash<P>>> {
 		let hashes = bytes_or_hash.into_iter()
 			.map(|x| match x {
 				hash::ExtrinsicOrHash::Hash(h) => Ok(h),
@@ -143,7 +205,11 @@ impl<B, E, P, RA> AuthorApi<ExHash<P>, BlockHash<P>> for Author<B, E, P, RA> whe
 		)
 	}
 
-	fn watch_extrinsic(&self, _metadata: Self::Metadata, subscriber: Subscriber<Status<ExHash<P>, BlockHash<P>>>, xt: Bytes) {
+	fn watch_extrinsic(&self,
+		_metadata: Self::Metadata,
+		subscriber: Subscriber<Status<ExHash<P>, BlockHash<P>>>,
+		xt: Bytes
+	) {
 		let submit = || -> Result<_> {
 			let best_block_hash = self.client.info().chain.best_hash;
 			let dxt = <<P as PoolChainApi>::Block as traits::Block>::Extrinsic::decode(&mut &xt[..])?;

substrate/core/rpc/src/author/tests.rs

@@ -23,9 +23,12 @@ use transaction_pool::{
 	txpool::Pool,
 	ChainApi,
 };
-use primitives::{H256, blake2_256, hexdisplay::HexDisplay};
+use primitives::{H256, blake2_256, hexdisplay::HexDisplay, traits::BareCryptoStore};
 use test_client::{self, AccountKeyring, runtime::{Extrinsic, Transfer}};
 use tokio::runtime;
+use std::collections::HashMap;
+use sr_primitives::KeyTypeId;
+use parking_lot::RwLock;
 
 fn uxt(sender: AccountKeyring, nonce: u64) -> Extrinsic {
 	let tx = Transfer {
@@ -37,14 +40,55 @@ fn uxt(sender: AccountKeyring, nonce: u64) -> Extrinsic {
 	tx.into_signed_tx()
 }
 
+#[derive(Default)]
+struct TestKeyStore {
+	keys: HashMap<KeyTypeId, HashMap<Vec<u8>, String>>,
+}
+
+impl BareCryptoStore for TestKeyStore {
+	fn sr25519_public_keys(&self, _id: KeyTypeId) -> Vec<sr25519::Public> { vec![] }
+	fn sr25519_generate_new(&mut self, _id: KeyTypeId, _seed: Option<&str>)
+		-> std::result::Result<sr25519::Public, String>
+	{
+		Err("unimplemented".into())
+	}
+	fn sr25519_key_pair(&self, _id: KeyTypeId, _pub_key: &sr25519::Public) -> Option<sr25519::Pair> {
+		None
+	}
+	fn ed25519_public_keys(&self, _id: KeyTypeId) -> Vec<ed25519::Public> { vec![] }
+	fn ed25519_generate_new(&mut self, _id: KeyTypeId, _seed: Option<&str>)
+		-> std::result::Result<ed25519::Public, String>
+	{
+		Err("unimplemented".into())
+	}
+	fn ed25519_key_pair(&self, _id: KeyTypeId, _pub_key: &ed25519::Public) -> Option<ed25519::Pair> {
+		None
+	}
+
+	fn insert_unknown(&mut self, key_type: KeyTypeId, suri: &str, public: &[u8])
+		-> std::result::Result<(), ()>
+	{
+		self.keys
+			.entry(key_type)
+			.or_default()
+			.insert(public.to_owned(), suri.to_owned());
+		Ok(())
+	}
+
+	fn password(&self) -> Option<&str> { None }
+}
+
 #[test]
 fn submit_transaction_should_not_cause_error() {
 	let runtime = runtime::Runtime::new().unwrap();
 	let client = Arc::new(test_client::new());
+	let keystore = TestKeyStore::default();
+	let keystore = Arc::new(RwLock::new(keystore));
 	let p = Author {
 		client: client.clone(),
 		pool: Arc::new(Pool::new(Default::default(), ChainApi::new(client))),
 		subscriptions: Subscriptions::new(Arc::new(runtime.executor())),
+		keystore: keystore.clone(),
 	};
 	let xt = uxt(AccountKeyring::Alice, 1).encode();
 	let h: H256 = blake2_256(&xt).into();
@@ -62,10 +106,12 @@ fn submit_transaction_should_not_cause_error() {
 fn submit_rich_transaction_should_not_cause_error() {
 	let runtime = runtime::Runtime::new().unwrap();
 	let client = Arc::new(test_client::new());
+	let keystore = Arc::new(RwLock::new(TestKeyStore::default()));
 	let p = Author {
 		client: client.clone(),
 		pool: Arc::new(Pool::new(Default::default(), ChainApi::new(client.clone()))),
 		subscriptions: Subscriptions::new(Arc::new(runtime.executor())),
+		keystore: keystore.clone(),
 	};
 	let xt = uxt(AccountKeyring::Alice, 0).encode();
 	let h: H256 = blake2_256(&xt).into();
@@ -85,10 +131,12 @@ fn should_watch_extrinsic() {
 	let mut runtime = runtime::Runtime::new().unwrap();
 	let client = Arc::new(test_client::new());
 	let pool = Arc::new(Pool::new(Default::default(), ChainApi::new(client.clone())));
+	let keystore = Arc::new(RwLock::new(TestKeyStore::default()));
 	let p = Author {
 		client,
 		pool: pool.clone(),
 		subscriptions: Subscriptions::new(Arc::new(runtime.executor())),
+		keystore: keystore.clone(),
 	};
 	let (subscriber, id_rx, data) = ::jsonrpc_pubsub::typed::Subscriber::new_test("test");
 
@@ -125,10 +173,12 @@ fn should_return_pending_extrinsics() {
 	let runtime = runtime::Runtime::new().unwrap();
 	let client = Arc::new(test_client::new());
 	let pool = Arc::new(Pool::new(Default::default(), ChainApi::new(client.clone())));
+	let keystore = Arc::new(RwLock::new(TestKeyStore::default()));
 	let p = Author {
 		client,
 		pool: pool.clone(),
 		subscriptions: Subscriptions::new(Arc::new(runtime.executor())),
+		keystore: keystore.clone(),
 	};
 	let ex = uxt(AccountKeyring::Alice, 0);
 	AuthorApi::submit_extrinsic(&p, ex.encode().into()).unwrap();
@@ -143,10 +193,12 @@ fn should_remove_extrinsics() {
 	let runtime = runtime::Runtime::new().unwrap();
 	let client = Arc::new(test_client::new());
 	let pool = Arc::new(Pool::new(Default::default(), ChainApi::new(client.clone())));
+	let keystore = Arc::new(RwLock::new(TestKeyStore::default()));
 	let p = Author {
 		client,
 		pool: pool.clone(),
 		subscriptions: Subscriptions::new(Arc::new(runtime.executor())),
+		keystore: keystore.clone(),
 	};
 	let ex1 = uxt(AccountKeyring::Alice, 0);
 	p.submit_extrinsic(ex1.encode().into()).unwrap();