Refactor key management (#3296)

* Add Call type to extensible transactions.

Cleanup some naming

* Merge Resource and BlockExhausted into just Exhausted

* Fix

* Another fix

* Call

* Some fixes

* Fix srml tests.

* Fix all tests.

* Refactor crypto so each application of it has its own type.

* Introduce new AuthorityProvider API into Aura

This will eventually allow for dynamic determination of authority
keys and avoid having to set them directly on CLI.

* Introduce authority determinator for Babe.

Experiment with modular consensus API.

* Work in progress to introduce KeyTypeId and avoid polluting API
with validator IDs

* Finish up drafting imonline

* Rework offchain workers API.

* Rework API implementation.

* Make it compile for wasm, simplify app_crypto.

* Fix compilation of im-online.

* Fix compilation of im-online.

* Fix more compilation errors.

* Make it compile.

* Fixing tests.

* Rewrite `keystore`

* Fix session tests

* Bring back `TryFrom`'s'

* Fix `srml-grandpa`

* Fix `srml-aura`

* Fix consensus babe

* More fixes

* Make service generate keys from dev_seed

* Build fixes

* Remove offchain tests

* More fixes and cleanups

* Fixes finality grandpa

* Fix `consensus-aura`

* Fix cli

* Fix `node-cli`

* Fix chain_spec builder

* Fix doc tests

* Add authority getter for grandpa.

* Test fix

* Fixes

* Make keystore accessible from the runtime

* Move app crypto to its own crate

* Update `Cargo.lock`

* Make the crypto stuff usable from the runtime

* Adds some runtime crypto tests

* Use last finalized block for grandpa authority

* Fix warning

* Adds `SessionKeys` runtime api

* Remove `FinalityPair` and `ConsensusPair`

* Minor governance tweaks to get it inline with docs.

* Make the governance be up to date with the docs.

* Build fixes.

* Generate the inital session keys

* Failing keystore is a hard error

* Make babe work again

* Fix grandpa

* Fix tests

* Disable `keystore` in consensus critical stuff

* Build fix.

* ImOnline supports multiple authorities at once.

* Update core/application-crypto/src/ed25519.rs

* Merge branch 'master' into gav-in-progress

* Remove unneeded code for now.

* Some `session` testing

* Support querying the public keys

* Cleanup offchain

* Remove warnings

* More cleanup

* Apply suggestions from code review

Co-Authored-By: Benjamin Kampmann <ben.kampmann@googlemail.com>

* More cleanups

* JSONRPC API for setting keys.

Also, rename traits::KeyStore* -> traits::BareCryptoStore*

* Bad merge

* Fix integration tests

* Fix test build

* Test fix

* Fixes

* Warnings

* Another warning

* Bump version.

substrate/core/service/src/components.rs

@@ -19,18 +19,21 @@
 use std::{sync::Arc, ops::Deref, ops::DerefMut};
 use serde::{Serialize, de::DeserializeOwned};
 use crate::chain_spec::ChainSpec;
+use keystore::KeyStorePtr;
 use client_db;
 use client::{self, Client, runtime_api};
-use crate::{error, Service, AuthorityKeyProvider};
+use crate::{error, Service};
 use consensus_common::{import_queue::ImportQueue, SelectChain};
-use network::{self, OnDemand, FinalityProofProvider, NetworkStateInfo, config::BoxFinalityProofRequestBuilder};
+use network::{
+	self, OnDemand, FinalityProofProvider, NetworkStateInfo, config::BoxFinalityProofRequestBuilder
+};
 use substrate_executor::{NativeExecutor, NativeExecutionDispatch};
 use transaction_pool::txpool::{self, Options as TransactionPoolOptions, Pool as TransactionPool};
 use sr_primitives::{
 	BuildStorage, traits::{Block as BlockT, Header as HeaderT, ProvideRuntimeApi}, generic::BlockId
 };
 use crate::config::Configuration;
-use primitives::{Blake2Hasher, H256, Pair};
+use primitives::{Blake2Hasher, H256, traits::BareCryptoStorePtr};
 use rpc::{self, apis::system::SystemInfo};
 use futures::{prelude::*, future::Executor};
 use futures03::channel::mpsc;
@@ -129,16 +132,6 @@ pub type ComponentOffchainStorage<C> = <
 /// Block type for `Components`
 pub type ComponentBlock<C> = <<C as Components>::Factory as ServiceFactory>::Block;
 
-/// ConsensusPair type for `Components`
-pub type ComponentConsensusPair<C> = <<C as Components>::Factory as ServiceFactory>::ConsensusPair;
-
-/// FinalityPair type for `Components`
-pub type ComponentFinalityPair<C> = <<C as Components>::Factory as ServiceFactory>::FinalityPair;
-
-/// AuthorityKeyProvider type for `Components`
-pub type ComponentAuthorityKeyProvider<C> =
-	AuthorityKeyProvider<ComponentBlock<C>, ComponentConsensusPair<C>, ComponentFinalityPair<C>>;
-
 /// Extrinsic hash type for `Components`
 pub type ComponentExHash<C> = <<C as Components>::TransactionPoolApi as txpool::ChainApi>::Hash;
 
@@ -152,6 +145,27 @@ pub type PoolApi<C> = <C as Components>::TransactionPoolApi;
 pub trait RuntimeGenesis: Serialize + DeserializeOwned + BuildStorage {}
 impl<T: Serialize + DeserializeOwned + BuildStorage> RuntimeGenesis for T {}
 
+/// Something that can create initial session keys from given seeds.
+pub trait InitialSessionKeys<C: Components> {
+	/// Generate the initial session keys for the given seeds.
+	fn generate_intial_session_keys(
+		client: Arc<ComponentClient<C>>,
+		seeds: Vec<String>,
+	) -> error::Result<()>;
+}
+
+impl<C: Components> InitialSessionKeys<Self> for C where
+	ComponentClient<C>: ProvideRuntimeApi,
+	<ComponentClient<C> as ProvideRuntimeApi>::Api: substrate_session::SessionKeys<ComponentBlock<C>>,
+{
+	fn generate_intial_session_keys(
+		client: Arc<ComponentClient<C>>,
+		seeds: Vec<String>,
+	) -> error::Result<()> {
+		substrate_session::generate_initial_session_keys(client, seeds).map_err(Into::into)
+	}
+}
+
 /// Something that can start the RPC service.
 pub trait StartRPC<C: Components> {
 	fn start_rpc(
@@ -160,6 +174,7 @@ pub trait StartRPC<C: Components> {
 		system_info: SystemInfo,
 		task_executor: TaskExecutor,
 		transaction_pool: Arc<TransactionPool<C::TransactionPoolApi>>,
+		keystore: KeyStorePtr,
 	) -> rpc::RpcHandler;
 }
 
@@ -173,11 +188,12 @@ impl<C: Components> StartRPC<Self> for C where
 		rpc_system_info: SystemInfo,
 		task_executor: TaskExecutor,
 		transaction_pool: Arc<TransactionPool<C::TransactionPoolApi>>,
+		keystore: KeyStorePtr,
 	) -> rpc::RpcHandler {
 		let subscriptions = rpc::apis::Subscriptions::new(task_executor.clone());
 		let chain = rpc::apis::chain::Chain::new(client.clone(), subscriptions.clone());
 		let state = rpc::apis::state::State::new(client.clone(), subscriptions.clone());
-		let author = rpc::apis::author::Author::new(client, transaction_pool, subscriptions);
+		let author = rpc::apis::author::Author::new(client, transaction_pool, subscriptions, keystore);
 		let system = rpc::apis::system::System::new(rpc_system_info, system_send_back);
 		rpc::rpc_handler::<ComponentBlock<C>, ComponentExHash<C>, _, _, _, _>(
 			state,
@@ -242,7 +258,6 @@ pub trait OffchainWorker<C: Components> {
 		offchain: &offchain::OffchainWorkers<
 			ComponentClient<C>,
 			ComponentOffchainStorage<C>,
-			ComponentAuthorityKeyProvider<C>,
 			ComponentBlock<C>
 		>,
 		pool: &Arc<TransactionPool<C::TransactionPoolApi>>,
@@ -259,7 +274,6 @@ impl<C: Components> OffchainWorker<Self> for C where
 		offchain: &offchain::OffchainWorkers<
 			ComponentClient<C>,
 			ComponentOffchainStorage<C>,
-			ComponentAuthorityKeyProvider<C>,
 			ComponentBlock<C>
 		>,
 		pool: &Arc<TransactionPool<C::TransactionPoolApi>>,
@@ -277,6 +291,7 @@ pub trait ServiceTrait<C: Components>:
 	+ StartRPC<C>
 	+ MaintainTransactionPool<C>
 	+ OffchainWorker<C>
+	+ InitialSessionKeys<C>
 {}
 impl<C: Components, T> ServiceTrait<C> for T where
 	T: Deref<Target = Service<C>>
@@ -285,6 +300,7 @@ impl<C: Components, T> ServiceTrait<C> for T where
 	+ StartRPC<C>
 	+ MaintainTransactionPool<C>
 	+ OffchainWorker<C>
+	+ InitialSessionKeys<C>
 {}
 
 /// Alias for a an implementation of `futures::future::Executor`.
@@ -294,10 +310,6 @@ pub type TaskExecutor = Arc<dyn Executor<Box<dyn Future<Item = (), Error = ()> +
 pub trait ServiceFactory: 'static + Sized {
 	/// Block type.
 	type Block: BlockT<Hash=H256>;
-	/// Consensus crypto type.
-	type ConsensusPair: Pair;
-	/// Finality crypto type.
-	type FinalityPair: Pair;
 	/// The type that implements the runtime API.
 	type RuntimeApi: Send + Sync;
 	/// Network protocol extensions.
@@ -412,6 +424,7 @@ pub trait Components: Sized + 'static {
 	fn build_client(
 		config: &FactoryFullConfiguration<Self::Factory>,
 		executor: CodeExecutor<Self::Factory>,
+		keystore: Option<BareCryptoStorePtr>,
 	) -> Result<
 		(
 			Arc<ComponentClient<Self>>,
@@ -498,11 +511,11 @@ impl<Factory: ServiceFactory> Components for FullComponents<Factory> {
 	fn build_client(
 		config: &FactoryFullConfiguration<Factory>,
 		executor: CodeExecutor<Self::Factory>,
-	)
-		-> Result<(
-			Arc<ComponentClient<Self>>,
-			Option<Arc<OnDemand<FactoryBlock<Self::Factory>>>>
-		), error::Error>
+		keystore: Option<BareCryptoStorePtr>,
+	) -> Result<
+			(Arc<ComponentClient<Self>>, Option<Arc<OnDemand<FactoryBlock<Self::Factory>>>>),
+			error::Error,
+		>
 	{
 		let db_settings = client_db::DatabaseSettings {
 			cache_size: config.database_cache_size.map(|u| u as usize),
@@ -512,12 +525,19 @@ impl<Factory: ServiceFactory> Components for FullComponents<Factory> {
 			path: config.database_path.clone(),
 			pruning: config.pruning.clone(),
 		};
-		Ok((Arc::new(client_db::new_client(
-			db_settings,
-			executor,
-			&config.chain_spec,
-			config.execution_strategies.clone(),
-		)?), None))
+
+		Ok((
+			Arc::new(
+				client_db::new_client(
+					db_settings,
+					executor,
+					&config.chain_spec,
+					config.execution_strategies.clone(),
+					keystore,
+				)?
+			),
+			None,
+		))
 	}
 
 	fn build_transaction_pool(
@@ -600,6 +620,7 @@ impl<Factory: ServiceFactory> Components for LightComponents<Factory> {
 	fn build_client(
 		config: &FactoryFullConfiguration<Factory>,
 		executor: CodeExecutor<Self::Factory>,
+		_: Option<BareCryptoStorePtr>,
 	)
 		-> Result<
 			(
@@ -615,9 +636,12 @@ impl<Factory: ServiceFactory> Components for LightComponents<Factory> {
 			path: config.database_path.clone(),
 			pruning: config.pruning.clone(),
 		};
+
 		let db_storage = client_db::light::LightStorage::new(db_settings)?;
 		let light_blockchain = client::light::new_light_blockchain(db_storage);
-		let fetch_checker = Arc::new(client::light::new_fetch_checker(light_blockchain.clone(), executor.clone()));
+		let fetch_checker = Arc::new(
+			client::light::new_fetch_checker(light_blockchain.clone(), executor.clone())
+		);
 		let fetcher = Arc::new(network::OnDemand::new(fetch_checker));
 		let client_backend = client::light::new_light_backend(light_blockchain, fetcher.clone());
 		let client = client::light::new_light(client_backend, fetcher.clone(), &config.chain_spec, executor)?;