Refactor key management (#3296)

* Add Call type to extensible transactions.

Cleanup some naming

* Merge Resource and BlockExhausted into just Exhausted

* Fix

* Another fix

* Call

* Some fixes

* Fix srml tests.

* Fix all tests.

* Refactor crypto so each application of it has its own type.

* Introduce new AuthorityProvider API into Aura

This will eventually allow for dynamic determination of authority
keys and avoid having to set them directly on CLI.

* Introduce authority determinator for Babe.

Experiment with modular consensus API.

* Work in progress to introduce KeyTypeId and avoid polluting API
with validator IDs

* Finish up drafting imonline

* Rework offchain workers API.

* Rework API implementation.

* Make it compile for wasm, simplify app_crypto.

* Fix compilation of im-online.

* Fix compilation of im-online.

* Fix more compilation errors.

* Make it compile.

* Fixing tests.

* Rewrite `keystore`

* Fix session tests

* Bring back `TryFrom`'s'

* Fix `srml-grandpa`

* Fix `srml-aura`

* Fix consensus babe

* More fixes

* Make service generate keys from dev_seed

* Build fixes

* Remove offchain tests

* More fixes and cleanups

* Fixes finality grandpa

* Fix `consensus-aura`

* Fix cli

* Fix `node-cli`

* Fix chain_spec builder

* Fix doc tests

* Add authority getter for grandpa.

* Test fix

* Fixes

* Make keystore accessible from the runtime

* Move app crypto to its own crate

* Update `Cargo.lock`

* Make the crypto stuff usable from the runtime

* Adds some runtime crypto tests

* Use last finalized block for grandpa authority

* Fix warning

* Adds `SessionKeys` runtime api

* Remove `FinalityPair` and `ConsensusPair`

* Minor governance tweaks to get it inline with docs.

* Make the governance be up to date with the docs.

* Build fixes.

* Generate the inital session keys

* Failing keystore is a hard error

* Make babe work again

* Fix grandpa

* Fix tests

* Disable `keystore` in consensus critical stuff

* Build fix.

* ImOnline supports multiple authorities at once.

* Update core/application-crypto/src/ed25519.rs

* Merge branch 'master' into gav-in-progress

* Remove unneeded code for now.

* Some `session` testing

* Support querying the public keys

* Cleanup offchain

* Remove warnings

* More cleanup

* Apply suggestions from code review

Co-Authored-By: Benjamin Kampmann <ben.kampmann@googlemail.com>

* More cleanups

* JSONRPC API for setting keys.

Also, rename traits::KeyStore* -> traits::BareCryptoStore*

* Bad merge

* Fix integration tests

* Fix test build

* Test fix

* Fixes

* Warnings

* Another warning

* Bump version.

substrate/core/state-machine/src/basic.rs

@@ -168,7 +168,12 @@ impl<H: Hasher> Externalities<H> for BasicExternalities where H::Out: Ord {
 	}
 
 	fn offchain(&mut self) -> Option<&mut dyn offchain::Externalities> {
-		warn!("Call to non-existent out offchain externalities set.");
+		warn!("Call to non-existent offchain externalities set.");
+		None
+	}
+
+	fn keystore(&self) -> Option<primitives::traits::BareCryptoStorePtr> {
+		warn!("Call to non-existent keystore.");
 		None
 	}
 }

substrate/core/state-machine/src/ext.rs

@@ -22,8 +22,7 @@ use crate::backend::Backend;
 use crate::changes_trie::{Storage as ChangesTrieStorage, build_changes_trie};
 use crate::{Externalities, OverlayedChanges, ChildStorageKey};
 use hash_db::Hasher;
-use primitives::offchain;
-use primitives::storage::well_known_keys::is_child_storage_key;
+use primitives::{offchain, storage::well_known_keys::is_child_storage_key, traits::BareCryptoStorePtr};
 use trie::{MemoryDB, default_child_trie_root};
 use trie::trie_types::Layout;
 
@@ -84,6 +83,8 @@ where
 	///
 	/// If None, some methods from the trait might not be supported.
 	offchain_externalities: Option<&'a mut O>,
+	/// The keystore that manages the keys of the node.
+	keystore: Option<BareCryptoStorePtr>,
 	/// Dummy usage of N arg.
 	_phantom: ::std::marker::PhantomData<N>,
 }
@@ -103,6 +104,7 @@ where
 		backend: &'a B,
 		changes_trie_storage: Option<&'a T>,
 		offchain_externalities: Option<&'a mut O>,
+		keystore: Option<BareCryptoStorePtr>,
 	) -> Self {
 		Ext {
 			overlay,
@@ -111,6 +113,7 @@ where
 			changes_trie_storage,
 			changes_trie_transaction: None,
 			offchain_externalities,
+			keystore,
 			_phantom: Default::default(),
 		}
 	}
@@ -333,6 +336,10 @@ where
 	fn offchain(&mut self) -> Option<&mut dyn offchain::Externalities> {
 		self.offchain_externalities.as_mut().map(|x| &mut **x as _)
 	}
+
+	fn keystore(&self) -> Option<BareCryptoStorePtr> {
+		self.keystore.clone()
+	}
 }
 
 #[cfg(test)]
@@ -375,7 +382,7 @@ mod tests {
 	fn storage_changes_root_is_none_when_storage_is_not_provided() {
 		let mut overlay = prepare_overlay_with_changes();
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &backend, None, None);
+		let mut ext = TestExt::new(&mut overlay, &backend, None, None, None);
 		assert_eq!(ext.storage_changes_root(Default::default()).unwrap(), None);
 	}
 
@@ -385,7 +392,7 @@ mod tests {
 		overlay.changes_trie_config = None;
 		let storage = TestChangesTrieStorage::with_blocks(vec![(100, Default::default())]);
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None);
+		let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None, None);
 		assert_eq!(ext.storage_changes_root(Default::default()).unwrap(), None);
 	}
 
@@ -394,11 +401,11 @@ mod tests {
 		let mut overlay = prepare_overlay_with_changes();
 		let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None);
-		let root = hex!("bb0c2ef6e1d36d5490f9766cfcc7dfe2a6ca804504c3bb206053890d6dd02376").into();
-
-		assert_eq!(ext.storage_changes_root(Default::default()).unwrap(),
-			Some(root));
+		let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None, None);
+		assert_eq!(
+			ext.storage_changes_root(Default::default()).unwrap(),
+			Some(hex!("bb0c2ef6e1d36d5490f9766cfcc7dfe2a6ca804504c3bb206053890d6dd02376").into()),
+		);
 	}
 
 	#[test]
@@ -407,10 +414,10 @@ mod tests {
 		overlay.prospective.top.get_mut(&vec![1]).unwrap().value = None;
 		let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None);
-		let root = hex!("96f5aae4690e7302737b6f9b7f8567d5bbb9eac1c315f80101235a92d9ec27f4").into();
-
-		assert_eq!(ext.storage_changes_root(Default::default()).unwrap(),
-			Some(root));
+		let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None, None);
+		assert_eq!(
+			ext.storage_changes_root(Default::default()).unwrap(),
+			Some(hex!("96f5aae4690e7302737b6f9b7f8567d5bbb9eac1c315f80101235a92d9ec27f4").into()),
+		);
 	}
 }

substrate/core/state-machine/src/lib.rs

@@ -25,6 +25,7 @@ use hash_db::Hasher;
 use codec::{Decode, Encode};
 use primitives::{
 	storage::well_known_keys, NativeOrEncoded, NeverNativeValue, offchain,
+	traits::BareCryptoStorePtr,
 };
 
 pub mod backend;
@@ -61,6 +62,7 @@ pub use proving_backend::{
 pub use trie_backend_essence::{TrieBackendStorage, Storage};
 pub use trie_backend::TrieBackend;
 
+
 /// A wrapper around a child storage key.
 ///
 /// This wrapper ensures that the child storage key is correct and properly used. It is
@@ -224,6 +226,9 @@ pub trait Externalities<H: Hasher> {
 
 	/// Returns offchain externalities extension if present.
 	fn offchain(&mut self) -> Option<&mut dyn offchain::Externalities>;
+
+	/// Returns the keystore.
+	fn keystore(&self) -> Option<BareCryptoStorePtr>;
 }
 
 /// An implementation of offchain extensions that should never be triggered.
@@ -247,53 +252,6 @@ impl offchain::Externalities for NeverOffchainExt {
 		unreachable!()
 	}
 
-	fn pubkey(
-		&self,
-		_key: offchain::CryptoKey,
-	) -> Result<Vec<u8>, ()> {
-		unreachable!()
-	}
-
-	fn new_crypto_key(
-		&mut self,
-		_crypto: offchain::CryptoKind,
-	) -> Result<offchain::CryptoKey, ()> {
-		unreachable!()
-	}
-
-	fn encrypt(
-		&mut self,
-		_key: offchain::CryptoKey,
-		_data: &[u8],
-	) -> Result<Vec<u8>, ()> {
-		unreachable!()
-	}
-
-	fn decrypt(
-		&mut self,
-		_key: offchain::CryptoKey,
-		_data: &[u8],
-	) -> Result<Vec<u8>, ()> {
-		unreachable!()
-	}
-
-	fn sign(
-		&mut self,
-		_key: offchain::CryptoKey,
-		_data: &[u8],
-	) -> Result<Vec<u8>, ()> {
-		unreachable!()
-	}
-
-	fn verify(
-		&mut self,
-		_key: offchain::CryptoKey,
-		_msg: &[u8],
-		_signature: &[u8],
-	) -> Result<bool, ()> {
-		unreachable!()
-	}
-
 	fn timestamp(&mut self) -> offchain::Timestamp {
 		unreachable!()
 	}
@@ -481,6 +439,7 @@ pub fn new<'a, H, N, B, T, O, Exec>(
 	exec: &'a Exec,
 	method: &'a str,
 	call_data: &'a [u8],
+	keystore: Option<BareCryptoStorePtr>,
 ) -> StateMachine<'a, H, N, B, T, O, Exec> {
 	StateMachine {
 		backend,
@@ -490,6 +449,7 @@ pub fn new<'a, H, N, B, T, O, Exec>(
 		exec,
 		method,
 		call_data,
+		keystore,
 		_hasher: PhantomData,
 	}
 }
@@ -503,6 +463,7 @@ pub struct StateMachine<'a, H, N, B, T, O, Exec> {
 	exec: &'a Exec,
 	method: &'a str,
 	call_data: &'a [u8],
+	keystore: Option<BareCryptoStorePtr>,
 	_hasher: PhantomData<(H, N)>,
 }
 
@@ -560,6 +521,7 @@ impl<'a, H, N, B, T, O, Exec> StateMachine<'a, H, N, B, T, O, Exec> where
 			self.backend,
 			self.changes_trie_storage,
 			self.offchain_ext.as_mut().map(|x| &mut **x),
+			self.keystore.clone(),
 		);
 		let (result, was_native) = self.exec.call(
 			&mut externalities,
@@ -707,6 +669,7 @@ pub fn prove_execution<B, H, Exec>(
 	exec: &Exec,
 	method: &str,
 	call_data: &[u8],
+	keystore: Option<BareCryptoStorePtr>,
 ) -> Result<(Vec<u8>, Vec<Vec<u8>>), Box<dyn Error>>
 where
 	B: Backend<H>,
@@ -716,7 +679,7 @@ where
 {
 	let trie_backend = backend.as_trie_backend()
 		.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<dyn Error>)?;
-	prove_execution_on_trie_backend(trie_backend, overlay, exec, method, call_data)
+	prove_execution_on_trie_backend(trie_backend, overlay, exec, method, call_data, keystore)
 }
 
 /// Prove execution using the given trie backend, overlayed changes, and call executor.
@@ -734,6 +697,7 @@ pub fn prove_execution_on_trie_backend<S, H, Exec>(
 	exec: &Exec,
 	method: &str,
 	call_data: &[u8],
+	keystore: Option<BareCryptoStorePtr>,
 ) -> Result<(Vec<u8>, Vec<Vec<u8>>), Box<dyn Error>>
 where
 	S: trie_backend_essence::TrieBackendStorage<H>,
@@ -750,6 +714,7 @@ where
 		exec,
 		method,
 		call_data,
+		keystore,
 		_hasher: PhantomData,
 	};
 	let (result, _, _) = sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
@@ -769,6 +734,7 @@ pub fn execution_proof_check<H, Exec>(
 	exec: &Exec,
 	method: &str,
 	call_data: &[u8],
+	keystore: Option<BareCryptoStorePtr>,
 ) -> Result<Vec<u8>, Box<dyn Error>>
 where
 	H: Hasher,
@@ -776,7 +742,7 @@ where
 	H::Out: Ord + 'static,
 {
 	let trie_backend = create_proof_check_backend::<H>(root.into(), proof)?;
-	execution_proof_check_on_trie_backend(&trie_backend, overlay, exec, method, call_data)
+	execution_proof_check_on_trie_backend(&trie_backend, overlay, exec, method, call_data, keystore)
 }
 
 /// Check execution proof on proving backend, generated by `prove_execution` call.
@@ -786,6 +752,7 @@ pub fn execution_proof_check_on_trie_backend<H, Exec>(
 	exec: &Exec,
 	method: &str,
 	call_data: &[u8],
+	keystore: Option<BareCryptoStorePtr>,
 ) -> Result<Vec<u8>, Box<dyn Error>>
 where
 	H: Hasher,
@@ -800,6 +767,7 @@ where
 		exec,
 		method,
 		call_data,
+		keystore,
 		_hasher: PhantomData,
 	};
 	sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
@@ -1050,6 +1018,7 @@ mod tests {
 			},
 			"test",
 			&[],
+			None,
 		).execute(
 			ExecutionStrategy::NativeWhenPossible
 		).unwrap().0, vec![66]);
@@ -1071,6 +1040,7 @@ mod tests {
 			},
 			"test",
 			&[],
+			None,
 		).execute(
 			ExecutionStrategy::NativeElseWasm
 		).unwrap().0, vec![66]);
@@ -1092,6 +1062,7 @@ mod tests {
 			},
 			"test",
 			&[],
+			None,
 		).execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
 			ExecutionManager::Both(|we, _ne| {
 				consensus_failed = true;
@@ -1114,13 +1085,26 @@ mod tests {
 
 		// fetch execution proof from 'remote' full node
 		let remote_backend = trie_backend::tests::test_trie();
-		let remote_root = remote_backend.storage_root(::std::iter::empty()).0;
-		let (remote_result, remote_proof) = prove_execution(remote_backend,
-			&mut Default::default(), &executor, "test", &[]).unwrap();
+		let remote_root = remote_backend.storage_root(std::iter::empty()).0;
+		let (remote_result, remote_proof) = prove_execution(
+			remote_backend,
+			&mut Default::default(),
+			&executor,
+			"test",
+			&[],
+			None,
+		).unwrap();
 
 		// check proof locally
-		let local_result = execution_proof_check::<Blake2Hasher, _>(remote_root, remote_proof,
-			&mut Default::default(), &executor, "test", &[]).unwrap();
+		let local_result = execution_proof_check::<Blake2Hasher, _>(
+			remote_root,
+			remote_proof,
+			&mut Default::default(),
+			&executor,
+			"test",
+			&[],
+			None,
+		).unwrap();
 
 		// check that both results are correct
 		assert_eq!(remote_result, vec![66]);
@@ -1151,7 +1135,13 @@ mod tests {
 
 		{
 			let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
-			let mut ext = Ext::new(&mut overlay, backend, Some(&changes_trie_storage), NeverOffchainExt::new());
+			let mut ext = Ext::new(
+				&mut overlay,
+				backend,
+				Some(&changes_trie_storage),
+				NeverOffchainExt::new(),
+				None,
+			);
 			ext.clear_prefix(b"ab");
 		}
 		overlay.commit_prospective();
@@ -1180,7 +1170,8 @@ mod tests {
 			&mut overlay,
 			backend,
 			Some(&changes_trie_storage),
-			NeverOffchainExt::new()
+			NeverOffchainExt::new(),
+			None,
 		);
 
 		ext.set_child_storage(
@@ -1252,41 +1243,47 @@ mod tests {
 
 	#[test]
 	fn cannot_change_changes_trie_config() {
-		assert!(new(
-			&trie_backend::tests::test_trie(),
-			Some(&InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new()),
-			NeverOffchainExt::new(),
-			&mut Default::default(),
-			&DummyCodeExecutor {
-				change_changes_trie_config: true,
-				native_available: false,
-				native_succeeds: true,
-				fallback_succeeds: true,
-			},
-			"test",
-			&[],
-		).execute(
-			ExecutionStrategy::NativeWhenPossible
-		).is_err());
+		assert!(
+			new(
+				&trie_backend::tests::test_trie(),
+				Some(&InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new()),
+				NeverOffchainExt::new(),
+				&mut Default::default(),
+				&DummyCodeExecutor {
+					change_changes_trie_config: true,
+					native_available: false,
+					native_succeeds: true,
+					fallback_succeeds: true,
+				},
+				"test",
+				&[],
+				None,
+			)
+			.execute(ExecutionStrategy::NativeWhenPossible)
+			.is_err()
+		);
 	}
 
 	#[test]
 	fn cannot_change_changes_trie_config_with_native_else_wasm() {
-		assert!(new(
-			&trie_backend::tests::test_trie(),
-			Some(&InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new()),
-			NeverOffchainExt::new(),
-			&mut Default::default(),
-			&DummyCodeExecutor {
-				change_changes_trie_config: true,
-				native_available: false,
-				native_succeeds: true,
-				fallback_succeeds: true,
-			},
-			"test",
-			&[],
-		).execute(
-			ExecutionStrategy::NativeElseWasm
-		).is_err());
+		assert!(
+			new(
+				&trie_backend::tests::test_trie(),
+				Some(&InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new()),
+				NeverOffchainExt::new(),
+				&mut Default::default(),
+				&DummyCodeExecutor {
+					change_changes_trie_config: true,
+					native_available: false,
+					native_succeeds: true,
+					fallback_succeeds: true,
+				},
+				"test",
+				&[],
+				None,
+			)
+			.execute(ExecutionStrategy::NativeElseWasm)
+			.is_err()
+		);
 	}
 }

substrate/core/state-machine/src/overlayed_changes.rs

@@ -371,6 +371,7 @@ mod tests {
 			&backend,
 			Some(&changes_trie_storage),
 			crate::NeverOffchainExt::new(),
+			None,
 		);
 		const ROOT: [u8; 32] = hex!("39245109cef3758c2eed2ccba8d9b370a917850af3824bc8348d505df2c298fa");
 

substrate/core/state-machine/src/testing.rs

@@ -25,8 +25,9 @@ use crate::changes_trie::{
 	build_changes_trie, InMemoryStorage as ChangesTrieInMemoryStorage,
 	BlockNumber as ChangesTrieBlockNumber,
 };
-use primitives::offchain;
-use primitives::storage::well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES};
+use primitives::{
+	storage::well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES}, traits::BareCryptoStorePtr, offchain
+};
 use codec::Encode;
 use super::{ChildStorageKey, Externalities, OverlayedChanges};
 
@@ -40,6 +41,7 @@ pub struct TestExternalities<H: Hasher, N: ChangesTrieBlockNumber> {
 	backend: InMemory<H>,
 	changes_trie_storage: ChangesTrieInMemoryStorage<H, N>,
 	offchain: Option<Box<dyn offchain::Externalities>>,
+	keystore: Option<BareCryptoStorePtr>,
 }
 
 impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N> {
@@ -84,6 +86,7 @@ impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N> {
 			changes_trie_storage: ChangesTrieInMemoryStorage::new(),
 			backend: backend.into(),
 			offchain: None,
+			keystore: None,
 		}
 	}
 
@@ -263,6 +266,10 @@ impl<H, N> Externalities<H> for TestExternalities<H, N>
 			.as_mut()
 			.map(|x| &mut **x as _)
 	}
+
+	fn keystore(&self) -> Option<BareCryptoStorePtr> {
+		self.keystore.clone()
+	}
 }
 
 #[cfg(test)]