Refactor key management (#3296)

* Add Call type to extensible transactions.

Cleanup some naming

* Merge Resource and BlockExhausted into just Exhausted

* Fix

* Another fix

* Call

* Some fixes

* Fix srml tests.

* Fix all tests.

* Refactor crypto so each application of it has its own type.

* Introduce new AuthorityProvider API into Aura

This will eventually allow for dynamic determination of authority
keys and avoid having to set them directly on CLI.

* Introduce authority determinator for Babe.

Experiment with modular consensus API.

* Work in progress to introduce KeyTypeId and avoid polluting API
with validator IDs

* Finish up drafting imonline

* Rework offchain workers API.

* Rework API implementation.

* Make it compile for wasm, simplify app_crypto.

* Fix compilation of im-online.

* Fix compilation of im-online.

* Fix more compilation errors.

* Make it compile.

* Fixing tests.

* Rewrite `keystore`

* Fix session tests

* Bring back `TryFrom`'s'

* Fix `srml-grandpa`

* Fix `srml-aura`

* Fix consensus babe

* More fixes

* Make service generate keys from dev_seed

* Build fixes

* Remove offchain tests

* More fixes and cleanups

* Fixes finality grandpa

* Fix `consensus-aura`

* Fix cli

* Fix `node-cli`

* Fix chain_spec builder

* Fix doc tests

* Add authority getter for grandpa.

* Test fix

* Fixes

* Make keystore accessible from the runtime

* Move app crypto to its own crate

* Update `Cargo.lock`

* Make the crypto stuff usable from the runtime

* Adds some runtime crypto tests

* Use last finalized block for grandpa authority

* Fix warning

* Adds `SessionKeys` runtime api

* Remove `FinalityPair` and `ConsensusPair`

* Minor governance tweaks to get it inline with docs.

* Make the governance be up to date with the docs.

* Build fixes.

* Generate the inital session keys

* Failing keystore is a hard error

* Make babe work again

* Fix grandpa

* Fix tests

* Disable `keystore` in consensus critical stuff

* Build fix.

* ImOnline supports multiple authorities at once.

* Update core/application-crypto/src/ed25519.rs

* Merge branch 'master' into gav-in-progress

* Remove unneeded code for now.

* Some `session` testing

* Support querying the public keys

* Cleanup offchain

* Remove warnings

* More cleanup

* Apply suggestions from code review

Co-Authored-By: Benjamin Kampmann <ben.kampmann@googlemail.com>

* More cleanups

* JSONRPC API for setting keys.

Also, rename traits::KeyStore* -> traits::BareCryptoStore*

* Bad merge

* Fix integration tests

* Fix test build

* Test fix

* Fixes

* Warnings

* Another warning

* Bump version.

substrate/node-template/Cargo.toml

@@ -16,7 +16,7 @@ ctrlc = { version = "3.0", features = ["termination"] }
 log = "0.4"
 tokio = "0.1"
 exit-future = "0.1"
-parking_lot = "0.8.0"
+parking_lot = "0.9.0"
 codec = { package = "parity-scale-codec", version = "1.0.0" }
 trie-root = "0.15.0"
 sr-io = { path = "../core/sr-io" }
@@ -28,6 +28,8 @@ inherents = { package = "substrate-inherents", path = "../core/inherents" }
 transaction-pool = { package = "substrate-transaction-pool", path = "../core/transaction-pool" }
 network = { package = "substrate-network", path = "../core/network" }
 consensus = { package = "substrate-consensus-aura", path = "../core/consensus/aura" }
+aura-primitives = { package = "substrate-consensus-aura-primitives", path = "../core/consensus/aura/primitives" }
+grandpa-primitives = { package = "substrate-finality-grandpa-primitives", path = "../core/finality-grandpa/primitives" }
 substrate-client = {  path = "../core/client" }
 basic-authorship = { package = "substrate-basic-authorship", path = "../core/basic-authorship" }
 node-template-runtime = { path = "runtime" }

substrate/node-template/runtime/Cargo.toml

@@ -13,6 +13,7 @@ runtime-io = { package = "sr-io", path = "../../core/sr-io", default_features =
 version = { package = "sr-version", path = "../../core/sr-version", default_features = false }
 support = { package = "srml-support", path = "../../srml/support", default_features = false }
 primitives = { package = "substrate-primitives", path = "../../core/primitives", default_features = false }
+substrate-session = { path = "../../core/session", default-features = false }
 balances = { package = "srml-balances", path = "../../srml/balances", default_features = false }
 aura = { package = "srml-aura", path = "../../srml/aura", default_features = false }
 executive = { package = "srml-executive", path = "../../srml/executive", default_features = false }
@@ -50,5 +51,6 @@ std = [
 	"safe-mix/std",
 	"consensus-aura/std",
 	"offchain-primitives/std",
+	"substrate-session/std",
 ]
 no_std = []

substrate/node-template/runtime/src/lib.rs

@@ -9,8 +9,11 @@
 include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
 
 use rstd::prelude::*;
-use primitives::{ed25519, sr25519, OpaqueMetadata};
-use sr_primitives::{ApplyResult, transaction_validity::TransactionValidity, generic, create_runtime_str};
+use primitives::{sr25519, OpaqueMetadata, crypto::key_types};
+use sr_primitives::{
+	ApplyResult, transaction_validity::TransactionValidity, generic, create_runtime_str,
+	impl_opaque_keys,
+};
 use sr_primitives::traits::{NumberFor, BlakeTwo256, Block as BlockT, StaticLookup, Verify, ConvertInto};
 use sr_primitives::weights::Weight;
 use client::{
@@ -30,10 +33,10 @@ pub use sr_primitives::{Permill, Perbill};
 pub use support::{StorageValue, construct_runtime, parameter_types};
 
 /// Alias to the signature scheme used for Aura authority signatures.
-pub type AuraSignature = ed25519::Signature;
+pub type AuraSignature = consensus_aura::sr25519::AuthoritySignature;
 
 /// The Ed25519 pub key of an session that belongs to an Aura authority of the chain.
-pub type AuraId = ed25519::Public;
+pub type AuraId = consensus_aura::sr25519::AuthorityId;
 
 /// Alias to pubkey that identifies an account on the chain.
 pub type AccountId = <AccountSignature as Verify>::Signer;
@@ -71,8 +74,13 @@ pub mod opaque {
 	pub type Block = generic::Block<Header, UncheckedExtrinsic>;
 	/// Opaque block identifier type.
 	pub type BlockId = generic::BlockId<Block>;
-	/// Opaque session key type.
-	pub type SessionKey = AuraId;
+
+	impl_opaque_keys! {
+		pub struct SessionKeys {
+			#[id(key_types::AURA)]
+			pub aura: AuraId,
+		}
+	}
 }
 
 /// This runtime version.
@@ -104,6 +112,8 @@ parameter_types! {
 impl system::Trait for Runtime {
 	/// The identifier used to distinguish between accounts.
 	type AccountId = AccountId;
+	/// The aggregated dispatch type that is available for extrinsics.
+	type Call = Call;
 	/// The lookup mechanism to get account ID from whatever is passed in dispatchers.
 	type Lookup = Indices;
 	/// The index type for storing how many extrinsics an account has signed.
@@ -152,6 +162,7 @@ impl indices::Trait for Runtime {
 parameter_types! {
 	pub const MinimumPeriod: u64 = 5000;
 }
+
 impl timestamp::Trait for Runtime {
 	/// A timestamp: milliseconds since the unix epoch.
 	type Moment = u64;
@@ -299,4 +310,11 @@ impl_runtime_apis! {
 			Executive::offchain_worker(n)
 		}
 	}
+
+	impl substrate_session::SessionKeys<Block> for Runtime {
+		fn generate_session_keys(seed: Option<Vec<u8>>) -> Vec<u8> {
+			let seed = seed.as_ref().map(|s| rstd::str::from_utf8(&s).expect("Seed is an utf8 string"));
+			opaque::SessionKeys::generate(seed)
+		}
+	}
 }

substrate/node-template/runtime/src/template.rs

@@ -93,6 +93,7 @@ mod tests {
 	}
 	impl system::Trait for Test {
 		type Origin = Origin;
+		type Call = ();
 		type Index = u64;
 		type BlockNumber = u64;
 		type Hash = H256;

substrate/node-template/src/chain_spec.rs

@@ -1,8 +1,9 @@
-use primitives::{ed25519, sr25519, Pair};
+use primitives::{sr25519, Pair};
 use node_template_runtime::{
 	AccountId, GenesisConfig, AuraConfig, BalancesConfig,
 	SudoConfig, IndicesConfig, SystemConfig, WASM_BINARY, AuraId
 };
+use aura_primitives::sr25519::AuthorityPair as AuraPair;
 use substrate_service;
 
 // Note this is the URL for the telemetry server
@@ -23,7 +24,7 @@ pub enum Alternative {
 }
 
 fn authority_key(s: &str) -> AuraId {
-	ed25519::Pair::from_string(&format!("//{}", s), None)
+	AuraPair::from_string(&format!("//{}", s), None)
 		.expect("static values are valid; qed")
 		.public()
 }

substrate/node-template/src/service.rs

@@ -15,7 +15,7 @@ use basic_authorship::ProposerFactory;
 use consensus::{import_queue, start_aura, AuraImportQueue, SlotDuration};
 use futures::prelude::*;
 use substrate_client::{self as client, LongestChain};
-use primitives::{ed25519::Pair, Pair as PairT};
+use primitives::{Pair as PairT};
 use inherents::InherentDataProviders;
 use network::{config::DummyFinalityProofRequestBuilder, construct_simple_protocol};
 use substrate_executor::native_executor_instance;
@@ -43,8 +43,6 @@ construct_simple_protocol! {
 construct_service_factory! {
 	struct Factory {
 		Block = Block,
-		ConsensusPair = Pair,
-		FinalityPair = Pair,
 		RuntimeApi = RuntimeApi,
 		NetworkProtocol = NodeProtocol { |config| Ok(NodeProtocol::new()) },
 		RuntimeDispatch = Executor,
@@ -68,7 +66,7 @@ construct_service_factory! {
 			},
 		AuthoritySetup = {
 			|service: Self::FullService| {
-				if let Some(key) = service.authority_key() {
+				if let Some(key) = None::<aura_primitives::sr25519::AuthorityPair> {
 					info!("Using authority key {}", key.public());
 					let proposer = ProposerFactory {
 						client: service.client(),
@@ -100,7 +98,7 @@ construct_service_factory! {
 			Self::Block,
 		>
 			{ |config: &mut FactoryFullConfiguration<Self> , client: Arc<FullClient<Self>>, _select_chain: Self::SelectChain| {
-					import_queue::<_, _, Pair>(
+					import_queue::<_, _, aura_primitives::sr25519::AuthorityPair>(
 						SlotDuration::get_or_compute(&*client)?,
 						Box::new(client.clone()),
 						None,
@@ -115,7 +113,7 @@ construct_service_factory! {
 		>
 			{ |config: &mut FactoryFullConfiguration<Self>, client: Arc<LightClient<Self>>| {
 					let fprb = Box::new(DummyFinalityProofRequestBuilder::default()) as Box<_>;
-					import_queue::<_, _, Pair>(
+					import_queue::<_, _, aura_primitives::sr25519::AuthorityPair>(
 						SlotDuration::get_or_compute(&*client)?,
 						Box::new(client.clone()),
 						None,