Refactor key management (#3296)

* Add Call type to extensible transactions.

Cleanup some naming

* Merge Resource and BlockExhausted into just Exhausted

* Fix

* Another fix

* Call

* Some fixes

* Fix srml tests.

* Fix all tests.

* Refactor crypto so each application of it has its own type.

* Introduce new AuthorityProvider API into Aura

This will eventually allow for dynamic determination of authority
keys and avoid having to set them directly on CLI.

* Introduce authority determinator for Babe.

Experiment with modular consensus API.

* Work in progress to introduce KeyTypeId and avoid polluting API
with validator IDs

* Finish up drafting imonline

* Rework offchain workers API.

* Rework API implementation.

* Make it compile for wasm, simplify app_crypto.

* Fix compilation of im-online.

* Fix compilation of im-online.

* Fix more compilation errors.

* Make it compile.

* Fixing tests.

* Rewrite `keystore`

* Fix session tests

* Bring back `TryFrom`'s'

* Fix `srml-grandpa`

* Fix `srml-aura`

* Fix consensus babe

* More fixes

* Make service generate keys from dev_seed

* Build fixes

* Remove offchain tests

* More fixes and cleanups

* Fixes finality grandpa

* Fix `consensus-aura`

* Fix cli

* Fix `node-cli`

* Fix chain_spec builder

* Fix doc tests

* Add authority getter for grandpa.

* Test fix

* Fixes

* Make keystore accessible from the runtime

* Move app crypto to its own crate

* Update `Cargo.lock`

* Make the crypto stuff usable from the runtime

* Adds some runtime crypto tests

* Use last finalized block for grandpa authority

* Fix warning

* Adds `SessionKeys` runtime api

* Remove `FinalityPair` and `ConsensusPair`

* Minor governance tweaks to get it inline with docs.

* Make the governance be up to date with the docs.

* Build fixes.

* Generate the inital session keys

* Failing keystore is a hard error

* Make babe work again

* Fix grandpa

* Fix tests

* Disable `keystore` in consensus critical stuff

* Build fix.

* ImOnline supports multiple authorities at once.

* Update core/application-crypto/src/ed25519.rs

* Merge branch 'master' into gav-in-progress

* Remove unneeded code for now.

* Some `session` testing

* Support querying the public keys

* Cleanup offchain

* Remove warnings

* More cleanup

* Apply suggestions from code review

Co-Authored-By: Benjamin Kampmann <ben.kampmann@googlemail.com>

* More cleanups

* JSONRPC API for setting keys.

Also, rename traits::KeyStore* -> traits::BareCryptoStore*

* Bad merge

* Fix integration tests

* Fix test build

* Test fix

* Fixes

* Warnings

* Another warning

* Bump version.
This commit is contained in:
Gavin Wood
2019-08-07 20:47:48 +02:00
committed by GitHub
parent a6a6779f01
commit 1a524b8207
160 changed files with 4467 additions and 2769 deletions
+18 -295
View File
@@ -21,174 +21,32 @@ use std::{
time::{SystemTime, Duration},
thread::sleep,
};
use client::backend::OffchainStorage;
use crate::AuthorityKeyProvider;
use futures::{Stream, Future, sync::mpsc};
use log::{info, debug, warn, error};
use network::{PeerId, Multiaddr, NetworkStateInfo};
use codec::{Encode, Decode};
use primitives::offchain::{
Timestamp,
HttpRequestId, HttpRequestStatus, HttpError,
Externalities as OffchainExt,
CryptoKind, CryptoKey,
StorageKind,
OpaqueNetworkState, OpaquePeerId, OpaqueMultiaddr,
};
use primitives::crypto::{Pair, Public, Protected};
use primitives::{ed25519, sr25519};
use sr_primitives::{
generic::BlockId,
traits::{self, Extrinsic},
Externalities as OffchainExt, HttpRequestId, Timestamp, HttpRequestStatus, HttpError,
OpaqueNetworkState, OpaquePeerId, OpaqueMultiaddr, StorageKind,
};
use sr_primitives::{generic::BlockId, traits::{self, Extrinsic}};
use transaction_pool::txpool::{Pool, ChainApi};
use network::NetworkStateInfo;
use network::{PeerId, Multiaddr};
/// A message between the offchain extension and the processing thread.
enum ExtMessage {
SubmitExtrinsic(Vec<u8>),
}
/// A persisted key seed.
#[derive(Encode, Decode)]
struct StoredKey {
kind: CryptoKind,
phrase: String,
}
impl StoredKey {
fn generate_with_phrase(kind: CryptoKind, password: Option<&str>) -> Self {
match kind {
CryptoKind::Ed25519 => {
let phrase = ed25519::Pair::generate_with_phrase(password).1;
Self { kind, phrase }
}
CryptoKind::Sr25519 => {
let phrase = sr25519::Pair::generate_with_phrase(password).1;
Self { kind, phrase }
}
}
}
fn to_local_key(&self, password: Option<&str>) -> Result<LocalKey, ()> {
match self.kind {
CryptoKind::Ed25519 => {
ed25519::Pair::from_phrase(&self.phrase, password)
.map(|x| LocalKey::Ed25519(x.0))
}
CryptoKind::Sr25519 => {
sr25519::Pair::from_phrase(&self.phrase, password)
.map(|x| LocalKey::Sr25519(x.0))
}
}
.map_err(|e| {
warn!("Error recovering Offchain Worker key. Password invalid? {:?}", e);
()
})
}
}
enum LocalKey {
Ed25519(ed25519::Pair),
Sr25519(sr25519::Pair),
}
impl LocalKey {
fn public(&self) -> Result<Vec<u8>, ()> {
match self {
LocalKey::Ed25519(pair) => Ok(pair.public().to_raw_vec()),
LocalKey::Sr25519(pair) => Ok(pair.public().to_raw_vec()),
}
}
fn sign(&self, data: &[u8]) -> Result<Vec<u8>, ()> {
match self {
LocalKey::Ed25519(pair) => {
let sig = pair.sign(data);
let bytes: &[u8] = sig.as_ref();
Ok(bytes.to_vec())
}
LocalKey::Sr25519(pair) => {
let sig = pair.sign(data);
let bytes: &[u8] = sig.as_ref();
Ok(bytes.to_vec())
}
}
}
fn verify(&self, msg: &[u8], signature: &[u8]) -> Result<bool, ()> {
match self {
LocalKey::Ed25519(pair) => {
Ok(ed25519::Pair::verify_weak(signature, msg, pair.public()))
}
LocalKey::Sr25519(pair) => {
Ok(sr25519::Pair::verify_weak(signature, msg, pair.public()))
}
}
}
}
/// A key.
enum Key<ConsensusPair, FinalityPair> {
LocalKey(LocalKey),
AuthorityKey(ConsensusPair),
FgAuthorityKey(FinalityPair),
}
impl<ConsensusPair: Pair, FinalityPair: Pair> Key<ConsensusPair, FinalityPair> {
fn public(&self) -> Result<Vec<u8>, ()> {
match self {
Key::LocalKey(local) => {
local.public()
}
Key::AuthorityKey(pair) => {
Ok(pair.public().to_raw_vec())
}
Key::FgAuthorityKey(pair) => {
Ok(pair.public().to_raw_vec())
}
}
}
fn sign(&self, data: &[u8]) -> Result<Vec<u8>, ()> {
match self {
Key::LocalKey(local) => {
local.sign(data)
}
Key::AuthorityKey(pair) => {
Ok(pair.sign(data).as_ref().to_vec())
}
Key::FgAuthorityKey(pair) => {
Ok(pair.sign(data).as_ref().to_vec())
}
}
}
fn verify(&self, msg: &[u8], signature: &[u8]) -> Result<bool, ()> {
match self {
Key::LocalKey(local) => {
local.verify(msg, signature)
}
Key::AuthorityKey(pair) => {
Ok(ConsensusPair::verify_weak(signature, msg, pair.public()))
}
Key::FgAuthorityKey(pair) => {
Ok(FinalityPair::verify_weak(signature, msg, pair.public()))
}
}
}
}
/// Asynchronous offchain API.
///
/// NOTE this is done to prevent recursive calls into the runtime (which are not supported currently).
pub(crate) struct Api<Storage, KeyProvider, Block: traits::Block> {
pub(crate) struct Api<Storage, Block: traits::Block> {
sender: mpsc::UnboundedSender<ExtMessage>,
db: Storage,
keys_password: Protected<String>,
key_provider: KeyProvider,
network_state: Arc<dyn NetworkStateInfo + Send + Sync>,
at: BlockId<Block>,
_at: BlockId<Block>,
}
fn unavailable_yet<R: Default>(name: &str) -> R {
@@ -199,59 +57,10 @@ fn unavailable_yet<R: Default>(name: &str) -> R {
const LOCAL_DB: &str = "LOCAL (fork-aware) DB";
const STORAGE_PREFIX: &[u8] = b"storage";
const KEYS_PREFIX: &[u8] = b"keys";
const NEXT_ID: &[u8] = b"crypto_key_id";
impl<Storage, KeyProvider, Block> Api<Storage, KeyProvider, Block> where
Storage: OffchainStorage,
KeyProvider: AuthorityKeyProvider<Block>,
Block: traits::Block,
{
fn password(&self) -> Option<&str> {
Some(self.keys_password.as_ref().as_str())
}
fn read_key(
&self,
key: CryptoKey,
) -> Result<Key<KeyProvider::ConsensusPair, KeyProvider::FinalityPair>, ()> {
match key {
CryptoKey::LocalKey { id, kind } => {
let key = self.db.get(KEYS_PREFIX, &id.encode())
.and_then(|key| StoredKey::decode(&mut &*key).ok())
.ok_or(())?;
if key.kind != kind {
warn!(
"Invalid crypto kind (got: {:?}, expected: {:?}), when requesting key {:?}",
key.kind,
kind,
id
);
return Err(())
}
Ok(Key::LocalKey(key.to_local_key(self.password())?))
}
CryptoKey::AuthorityKey => {
let key = self.key_provider
.authority_key(&self.at)
.ok_or(())?;
Ok(Key::AuthorityKey(key))
}
CryptoKey::FgAuthorityKey => {
let key = self.key_provider
.fg_authority_key(&self.at)
.ok_or(())?;
Ok(Key::FgAuthorityKey(key))
}
}
}
}
impl<Storage, KeyProvider, Block> OffchainExt for Api<Storage, KeyProvider, Block>
impl<Storage, Block> OffchainExt for Api<Storage, Block>
where
Storage: OffchainStorage,
KeyProvider: AuthorityKeyProvider<Block>,
Block: traits::Block,
{
fn submit_transaction(&mut self, ext: Vec<u8>) -> Result<(), ()> {
@@ -261,30 +70,6 @@ where
.map_err(|_| ())
}
fn new_crypto_key(&mut self, kind: CryptoKind) -> Result<CryptoKey, ()> {
let key = StoredKey::generate_with_phrase(kind, self.password());
let (id, id_encoded) = loop {
let encoded = self.db.get(KEYS_PREFIX, NEXT_ID);
let encoded_slice = encoded.as_ref().map(|x| x.as_slice());
let new_id = encoded_slice.and_then(|mut x| u16::decode(&mut x).ok()).unwrap_or_default()
.checked_add(1)
.ok_or(())?;
let new_id_encoded = new_id.encode();
if self.db.compare_and_set(KEYS_PREFIX, NEXT_ID, encoded_slice, &new_id_encoded) {
break (new_id, new_id_encoded);
}
};
self.db.set(KEYS_PREFIX, &id_encoded, &key.encode());
Ok(CryptoKey::LocalKey { id, kind })
}
fn pubkey(&self, key: CryptoKey) -> Result<Vec<u8>, ()> {
self.read_key(key)?.public()
}
fn network_state(&self) -> Result<OpaqueNetworkState, ()> {
let external_addresses = self.network_state.external_addresses();
@@ -295,25 +80,6 @@ where
Ok(OpaqueNetworkState::from(state))
}
fn encrypt(&mut self, _key: CryptoKey, _data: &[u8]) -> Result<Vec<u8>, ()> {
unavailable_yet::<()>("encrypt");
Err(())
}
fn decrypt(&mut self, _key: CryptoKey, _data: &[u8]) -> Result<Vec<u8>, ()> {
unavailable_yet::<()>("decrypt");
Err(())
}
fn sign(&mut self, key: CryptoKey, data: &[u8]) -> Result<Vec<u8>, ()> {
self.read_key(key)?.sign(data)
}
fn verify(&mut self, key: CryptoKey, msg: &[u8], signature: &[u8]) -> Result<bool, ()> {
self.read_key(key)?.verify(msg, signature)
}
fn timestamp(&mut self) -> Timestamp {
let now = SystemTime::now();
let epoch_duration = now.duration_since(SystemTime::UNIX_EPOCH);
@@ -506,23 +272,19 @@ pub(crate) struct AsyncApi<A: ChainApi> {
impl<A: ChainApi> AsyncApi<A> {
/// Creates new Offchain extensions API implementation an the asynchronous processing part.
pub fn new<S: OffchainStorage, P: AuthorityKeyProvider<A::Block>>(
pub fn new<S: OffchainStorage>(
transaction_pool: Arc<Pool<A>>,
db: S,
keys_password: Protected<String>,
key_provider: P,
at: BlockId<A::Block>,
network_state: Arc<dyn NetworkStateInfo + Send + Sync>,
) -> (Api<S, P, A::Block>, AsyncApi<A>) {
) -> (Api<S, A::Block>, AsyncApi<A>) {
let (sender, rx) = mpsc::unbounded();
let api = Api {
sender,
db,
keys_password,
key_provider,
network_state,
at,
_at: at,
};
let async_api = AsyncApi {
@@ -571,7 +333,6 @@ mod tests {
use std::convert::TryFrom;
use sr_primitives::traits::Zero;
use client_db::offchain::LocalStorage;
use crate::tests::TestProvider;
use network::PeerId;
use test_client::runtime::Block;
@@ -587,7 +348,7 @@ mod tests {
}
}
fn offchain_api() -> (Api<LocalStorage, TestProvider<Block>, Block>, AsyncApi<impl ChainApi>) {
fn offchain_api() -> (Api<LocalStorage, Block>, AsyncApi<impl ChainApi>) {
let _ = env_logger::try_init();
let db = LocalStorage::new_test();
let client = Arc::new(test_client::new());
@@ -596,7 +357,12 @@ mod tests {
);
let mock = Arc::new(MockNetworkStateInfo());
AsyncApi::new(pool, db, "pass".to_owned().into(), TestProvider::default(), BlockId::Number(Zero::zero()), mock)
AsyncApi::new(
pool,
db,
BlockId::Number(Zero::zero()),
mock,
)
}
#[test]
@@ -680,49 +446,6 @@ mod tests {
assert_eq!(api.local_storage_get(kind, key), Some(b"value".to_vec()));
}
#[test]
fn should_create_a_new_key_and_sign_and_verify_stuff() {
let test = |kind: CryptoKind| {
// given
let mut api = offchain_api().0;
let msg = b"Hello world!";
// when
let key = api.new_crypto_key(kind).unwrap();
let signature = api.sign(key, msg).unwrap();
// then
let res = api.verify(key, msg, &signature).unwrap();
assert_eq!(res, true);
let res = api.verify(key, msg, &[]).unwrap();
assert_eq!(res, false);
let res = api.verify(key, b"Different msg", &signature).unwrap();
assert_eq!(res, false);
};
test(CryptoKind::Ed25519);
test(CryptoKind::Sr25519);
}
#[test]
fn should_sign_and_verify_with_authority_key() {
// given
let mut api = offchain_api().0;
api.key_provider.ed_key = Some(ed25519::Pair::generate().0);
let msg = b"Hello world!";
// when
let signature = api.sign(CryptoKey::AuthorityKey, msg).unwrap();
// then
let res = api.verify(CryptoKey::AuthorityKey, msg, &signature).unwrap();
assert_eq!(res, true);
let res = api.verify(CryptoKey::AuthorityKey, msg, &[]).unwrap();
assert_eq!(res, false);
let res = api.verify(CryptoKey::AuthorityKey, b"Different msg", &signature).unwrap();
assert_eq!(res, false);
}
#[test]
fn should_convert_network_states() {
// given
+14 -89
View File
@@ -40,18 +40,12 @@ use std::{
};
use client::runtime_api::ApiExt;
use log::{debug, warn};
use primitives::{
ExecutionContext,
crypto,
};
use sr_primitives::{
generic::BlockId,
traits::{self, ProvideRuntimeApi},
};
use futures::future::Future;
use transaction_pool::txpool::{Pool, ChainApi};
use log::{debug, warn};
use network::NetworkStateInfo;
use primitives::ExecutionContext;
use sr_primitives::{generic::BlockId, traits::{self, ProvideRuntimeApi}};
use transaction_pool::txpool::{Pool, ChainApi};
mod api;
@@ -59,61 +53,27 @@ pub mod testing;
pub use offchain_primitives::OffchainWorkerApi;
/// Provides currently configured authority key.
pub trait AuthorityKeyProvider<Block: traits::Block>: Clone + 'static {
/// The crypto used by the block authoring algorithm.
type ConsensusPair: crypto::Pair;
/// The crypto used by the finality gadget.
type FinalityPair: crypto::Pair;
/// Returns currently configured authority key.
fn authority_key(&self, block_id: &BlockId<Block>) -> Option<Self::ConsensusPair>;
/// Returns currently configured finality gadget authority key.
fn fg_authority_key(&self, block_id: &BlockId<Block>) -> Option<Self::FinalityPair>;
}
/// An offchain workers manager.
pub struct OffchainWorkers<
Client,
Storage,
KeyProvider,
Block: traits::Block,
> {
pub struct OffchainWorkers<Client, Storage, Block: traits::Block> {
client: Arc<Client>,
db: Storage,
authority_key: KeyProvider,
keys_password: crypto::Protected<String>,
_block: PhantomData<Block>,
}
impl<Client, Storage, KeyProvider, Block: traits::Block> OffchainWorkers<
Client,
Storage,
KeyProvider,
Block,
> {
impl<Client, Storage, Block: traits::Block> OffchainWorkers<Client, Storage, Block> {
/// Creates new `OffchainWorkers`.
pub fn new(
client: Arc<Client>,
db: Storage,
authority_key: KeyProvider,
keys_password: crypto::Protected<String>,
) -> Self {
pub fn new(client: Arc<Client>, db: Storage) -> Self {
Self {
client,
db,
authority_key,
keys_password,
_block: PhantomData,
}
}
}
impl<Client, Storage, KeyProvider, Block: traits::Block> fmt::Debug for OffchainWorkers<
impl<Client, Storage, Block: traits::Block> fmt::Debug for OffchainWorkers<
Client,
Storage,
KeyProvider,
Block,
> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
@@ -121,16 +81,14 @@ impl<Client, Storage, KeyProvider, Block: traits::Block> fmt::Debug for Offchain
}
}
impl<Client, Storage, KeyProvider, Block> OffchainWorkers<
impl<Client, Storage, Block> OffchainWorkers<
Client,
Storage,
KeyProvider,
Block,
> where
Block: traits::Block,
Client: ProvideRuntimeApi + Send + Sync + 'static,
Client::Api: OffchainWorkerApi<Block>,
KeyProvider: AuthorityKeyProvider<Block> + Send,
Storage: client::backend::OffchainStorage + 'static,
{
/// Start the offchain workers after given block.
@@ -152,8 +110,6 @@ impl<Client, Storage, KeyProvider, Block> OffchainWorkers<
let (api, runner) = api::AsyncApi::new(
pool.clone(),
self.db.clone(),
self.keys_password.clone(),
self.authority_key.clone(),
at.clone(),
network_state.clone(),
);
@@ -167,7 +123,7 @@ impl<Client, Storage, KeyProvider, Block> OffchainWorkers<
let run = runtime.offchain_worker_with_context(
&at,
ExecutionContext::OffchainWorker(api),
number
number,
);
if let Err(e) = run {
log::error!("Error running offchain workers at {:?}: {:?}", at, e);
@@ -197,7 +153,6 @@ fn spawn_worker(f: impl FnOnce() -> () + Send + 'static) {
mod tests {
use super::*;
use futures::Future;
use primitives::{ed25519, sr25519};
use network::{Multiaddr, PeerId};
struct MockNetworkStateInfo();
@@ -212,49 +167,19 @@ mod tests {
}
}
#[derive(Clone)]
pub(crate) struct TestProvider<Block> {
_marker: PhantomData<Block>,
pub(crate) sr_key: Option<sr25519::Pair>,
pub(crate) ed_key: Option<ed25519::Pair>,
}
impl<Block: traits::Block> Default for TestProvider<Block> {
fn default() -> Self {
Self {
_marker: PhantomData,
sr_key: None,
ed_key: None,
}
}
}
impl<Block: traits::Block> AuthorityKeyProvider<Block> for TestProvider<Block> {
type ConsensusPair = ed25519::Pair;
type FinalityPair = sr25519::Pair;
fn authority_key(&self, _: &BlockId<Block>) -> Option<Self::ConsensusPair> {
self.ed_key.clone()
}
fn fg_authority_key(&self, _: &BlockId<Block>) -> Option<Self::FinalityPair> {
self.sr_key.clone()
}
}
#[test]
fn should_call_into_runtime_and_produce_extrinsic() {
// given
let _ = env_logger::try_init();
let runtime = tokio::runtime::Runtime::new().unwrap();
let client = Arc::new(test_client::new());
let pool = Arc::new(Pool::new(Default::default(), ::transaction_pool::ChainApi::new(client.clone())));
let pool = Arc::new(Pool::new(Default::default(), transaction_pool::ChainApi::new(client.clone())));
let db = client_db::offchain::LocalStorage::new_test();
let mock = Arc::new(MockNetworkStateInfo());
let network_state = Arc::new(MockNetworkStateInfo());
// when
let offchain = OffchainWorkers::new(client, db, TestProvider::default(), "".to_owned().into());
runtime.executor().spawn(offchain.on_block_imported(&0u64, &pool, mock.clone()));
let offchain = OffchainWorkers::new(client, db);
runtime.executor().spawn(offchain.on_block_imported(&0u64, &pool, network_state.clone()));
// then
runtime.shutdown_on_idle().wait().unwrap();
-43
View File
@@ -28,8 +28,6 @@ use primitives::offchain::{
HttpRequestId as RequestId,
HttpRequestStatus as RequestStatus,
Timestamp,
CryptoKind,
CryptoKey,
StorageKind,
OpaqueNetworkState,
};
@@ -144,47 +142,6 @@ impl offchain::Externalities for TestOffchainExt {
unimplemented!("not needed in tests so far")
}
fn pubkey(&self, _key: CryptoKey) -> Result<Vec<u8>, ()> {
unimplemented!("not needed in tests so far")
}
fn new_crypto_key(&mut self, _crypto: CryptoKind) -> Result<CryptoKey, ()> {
unimplemented!("not needed in tests so far")
}
fn encrypt(
&mut self,
_key: CryptoKey,
_data: &[u8],
) -> Result<Vec<u8>, ()> {
unimplemented!("not needed in tests so far")
}
fn decrypt(
&mut self,
_key: CryptoKey,
_data: &[u8],
) -> Result<Vec<u8>, ()> {
unimplemented!("not needed in tests so far")
}
fn sign(
&mut self,
_key: CryptoKey,
_data: &[u8],
) -> Result<Vec<u8>, ()> {
unimplemented!("not needed in tests so far")
}
fn verify(
&mut self,
_key: CryptoKey,
_msg: &[u8],
_signature: &[u8],
) -> Result<bool, ()> {
unimplemented!("not needed in tests so far")
}
fn timestamp(&mut self) -> Timestamp {
unimplemented!("not needed in tests so far")
}