Run cargo fmt on the whole code base (#9394)

* Run cargo fmt on the whole code base

* Second run

* Add CI check

* Fix compilation

* More unnecessary braces

* Handle weights

* Use --all

* Use correct attributes...

* Fix UI tests

* AHHHHHHHHH

* 🤦

* Docs

* Fix compilation

* 🤷

* Please stop

* 🤦 x 2

* More

* make rustfmt.toml consistent with polkadot

Co-authored-by: André Silva <andrerfosilva@gmail.com>
This commit is contained in:
Bastian Köcher
2021-07-21 16:32:32 +02:00
committed by GitHub
parent d451c38c1c
commit 7b56ab15b4
1010 changed files with 53339 additions and 51208 deletions
+1 -1
View File
@@ -26,5 +26,5 @@ pub enum Error {
AllocatorOutOfSpace,
/// Some other error occurred.
#[error("Other: {0}")]
Other(&'static str)
Other(&'static str),
}
+22 -16
View File
@@ -68,8 +68,12 @@
//! sizes.
use crate::Error;
use std::{mem, convert::{TryFrom, TryInto}, ops::{Range, Index, IndexMut}};
use sp_wasm_interface::{Pointer, WordSize};
use std::{
convert::{TryFrom, TryInto},
mem,
ops::{Index, IndexMut, Range},
};
/// The minimal alignment guaranteed by this allocator.
///
@@ -139,7 +143,7 @@ impl Order {
fn from_size(size: u32) -> Result<Self, Error> {
let clamped_size = if size > MAX_POSSIBLE_ALLOCATION {
log::warn!(target: LOG_TARGET, "going to fail due to allocating {:?}", size);
return Err(Error::RequestedAllocationTooLarge);
return Err(Error::RequestedAllocationTooLarge)
} else if size < MIN_POSSIBLE_ALLOCATION {
MIN_POSSIBLE_ALLOCATION
} else {
@@ -216,7 +220,6 @@ impl Link {
/// ```
///
/// ## Occupied header
///
/// ```ignore
/// 64 32 0
// +--------------+-------------------+
@@ -290,9 +293,7 @@ struct FreeLists {
impl FreeLists {
/// Creates the free empty lists.
fn new() -> Self {
Self {
heads: [Link::Nil; N_ORDERS]
}
Self { heads: [Link::Nil; N_ORDERS] }
}
/// Replaces a given link for the specified order and returns the old one.
@@ -397,15 +398,11 @@ impl FreeingBumpHeapAllocator {
self.free_lists[order] = next_free;
header_ptr
}
},
Link::Nil => {
// Corresponding free list is empty. Allocate a new item.
Self::bump(
&mut self.bumper,
order.size() + HEADER_SIZE,
mem.size(),
)?
}
Self::bump(&mut self.bumper, order.size() + HEADER_SIZE, mem.size())?
},
};
// Write the order in the occupied header.
@@ -440,7 +437,11 @@ impl FreeingBumpHeapAllocator {
///
/// - `mem` - a slice representing the linear memory on which this allocator operates.
/// - `ptr` - pointer to the allocated chunk
pub fn deallocate<M: Memory + ?Sized>(&mut self, mem: &mut M, ptr: Pointer<u8>) -> Result<(), Error> {
pub fn deallocate<M: Memory + ?Sized>(
&mut self,
mem: &mut M,
ptr: Pointer<u8>,
) -> Result<(), Error> {
if self.poisoned {
return Err(error("the allocator has been poisoned"))
}
@@ -480,8 +481,13 @@ impl FreeingBumpHeapAllocator {
/// the operation would exhaust the heap.
fn bump(bumper: &mut u32, size: u32, heap_end: u32) -> Result<u32, Error> {
if *bumper + size > heap_end {
log::error!(target: LOG_TARGET, "running out of space with current bumper {}, mem size {}", bumper, heap_end);
return Err(Error::AllocatorOutOfSpace);
log::error!(
target: LOG_TARGET,
"running out of space with current bumper {}, mem size {}",
bumper,
heap_end
);
return Err(Error::AllocatorOutOfSpace)
}
let res = *bumper;
+1 -1
View File
@@ -25,5 +25,5 @@
mod error;
mod freeing_bump;
pub use freeing_bump::FreeingBumpHeapAllocator;
pub use error::Error;
pub use freeing_bump::FreeingBumpHeapAllocator;
+91 -73
View File
@@ -18,30 +18,32 @@
//! Substrate Client data backend
use std::sync::Arc;
use std::collections::{HashMap, HashSet};
use sp_core::ChangesTrieConfigurationRange;
use sp_core::offchain::OffchainStorage;
use sp_runtime::{generic::BlockId, Justification, Justifications, Storage};
use sp_runtime::traits::{Block as BlockT, NumberFor, HashFor};
use sp_state_machine::{
ChangesTrieState, ChangesTrieStorage as StateChangesTrieStorage, ChangesTrieTransaction,
StorageCollection, ChildStorageCollection, OffchainChangesCollection, IndexOperation,
};
use sp_storage::{StorageData, StorageKey, PrefixedStorageKey, ChildInfo};
use crate::{
blockchain::{
Backend as BlockchainBackend, well_known_cache_keys
},
blockchain::{well_known_cache_keys, Backend as BlockchainBackend},
light::RemoteBlockchain,
UsageInfo,
};
use parking_lot::RwLock;
use sp_blockchain;
use sp_consensus::BlockOrigin;
use parking_lot::RwLock;
use sp_core::{offchain::OffchainStorage, ChangesTrieConfigurationRange};
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, HashFor, NumberFor},
Justification, Justifications, Storage,
};
use sp_state_machine::{
ChangesTrieState, ChangesTrieStorage as StateChangesTrieStorage, ChangesTrieTransaction,
ChildStorageCollection, IndexOperation, OffchainChangesCollection, StorageCollection,
};
use sp_storage::{ChildInfo, PrefixedStorageKey, StorageData, StorageKey};
use std::{
collections::{HashMap, HashSet},
sync::Arc,
};
pub use sp_state_machine::Backend as StateBackend;
pub use sp_consensus::ImportedState;
pub use sp_state_machine::Backend as StateBackend;
use std::marker::PhantomData;
/// Extracts the state backend type for the given backend.
@@ -90,16 +92,17 @@ pub fn apply_aux<'a, 'b: 'a, 'c: 'a, B, Block, D, I>(
insert: I,
delete: D,
) -> sp_blockchain::Result<()>
where
Block: BlockT,
B: Backend<Block>,
I: IntoIterator<Item=&'a(&'c [u8], &'c [u8])>,
D: IntoIterator<Item=&'a &'b [u8]>,
where
Block: BlockT,
B: Backend<Block>,
I: IntoIterator<Item = &'a (&'c [u8], &'c [u8])>,
D: IntoIterator<Item = &'a &'b [u8]>,
{
operation.op.insert_aux(
insert.into_iter()
insert
.into_iter()
.map(|(k, v)| (k.to_vec(), Some(v.to_vec())))
.chain(delete.into_iter().map(|k| (k.to_vec(), None)))
.chain(delete.into_iter().map(|k| (k.to_vec(), None))),
)
}
@@ -165,7 +168,11 @@ pub trait BlockImportOperation<Block: BlockT> {
/// Set genesis state. If `commit` is `false` the state is saved in memory, but is not written
/// to the database.
fn set_genesis_state(&mut self, storage: Storage, commit: bool) -> sp_blockchain::Result<Block::Hash>;
fn set_genesis_state(
&mut self,
storage: Storage,
commit: bool,
) -> sp_blockchain::Result<Block::Hash>;
/// Inject storage data into the database replacing any existing data.
fn reset_storage(&mut self, storage: Storage) -> sp_blockchain::Result<Block::Hash>;
@@ -182,7 +189,7 @@ pub trait BlockImportOperation<Block: BlockT> {
&mut self,
_offchain_update: OffchainChangesCollection,
) -> sp_blockchain::Result<()> {
Ok(())
Ok(())
}
/// Inject changes trie data into the database.
@@ -195,7 +202,8 @@ pub trait BlockImportOperation<Block: BlockT> {
///
/// Values are `None` if should be deleted.
fn insert_aux<I>(&mut self, ops: I) -> sp_blockchain::Result<()>
where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>;
where
I: IntoIterator<Item = (Vec<u8>, Option<Vec<u8>>)>;
/// Mark a block as finalized.
fn mark_finalized(
@@ -209,16 +217,17 @@ pub trait BlockImportOperation<Block: BlockT> {
fn mark_head(&mut self, id: BlockId<Block>) -> sp_blockchain::Result<()>;
/// Add a transaction index operation.
fn update_transaction_index(&mut self, index: Vec<IndexOperation>) -> sp_blockchain::Result<()>;
fn update_transaction_index(&mut self, index: Vec<IndexOperation>)
-> sp_blockchain::Result<()>;
}
/// Interface for performing operations on the backend.
pub trait LockImportRun<Block: BlockT, B: Backend<Block>> {
/// Lock the import lock, and run operations inside.
fn lock_import_and_run<R, Err, F>(&self, f: F) -> Result<R, Err>
where
F: FnOnce(&mut ClientImportOperation<Block, B>) -> Result<R, Err>,
Err: From<sp_blockchain::Error>;
where
F: FnOnce(&mut ClientImportOperation<Block, B>) -> Result<R, Err>,
Err: From<sp_blockchain::Error>;
}
/// Finalize Facilities
@@ -270,9 +279,13 @@ pub trait AuxStore {
'a,
'b: 'a,
'c: 'a,
I: IntoIterator<Item=&'a(&'c [u8], &'c [u8])>,
D: IntoIterator<Item=&'a &'b [u8]>,
>(&self, insert: I, delete: D) -> sp_blockchain::Result<()>;
I: IntoIterator<Item = &'a (&'c [u8], &'c [u8])>,
D: IntoIterator<Item = &'a &'b [u8]>,
>(
&self,
insert: I,
delete: D,
) -> sp_blockchain::Result<()>;
/// Query auxiliary data from key-value store.
fn get_aux(&self, key: &[u8]) -> sp_blockchain::Result<Option<Vec<u8>>>;
@@ -287,16 +300,10 @@ pub struct KeyIterator<'a, State, Block> {
_phantom: PhantomData<Block>,
}
impl <'a, State, Block> KeyIterator<'a, State, Block> {
impl<'a, State, Block> KeyIterator<'a, State, Block> {
/// create a KeyIterator instance
pub fn new(state: State, prefix: Option<&'a StorageKey>, current_key: Vec<u8>) -> Self {
Self {
state,
child_storage: None,
prefix,
current_key,
_phantom: PhantomData,
}
Self { state, child_storage: None, prefix, current_key, _phantom: PhantomData }
}
/// Create a `KeyIterator` instance for a child storage.
@@ -306,17 +313,12 @@ impl <'a, State, Block> KeyIterator<'a, State, Block> {
prefix: Option<&'a StorageKey>,
current_key: Vec<u8>,
) -> Self {
Self {
state,
child_storage: Some(child_info),
prefix,
current_key,
_phantom: PhantomData,
}
Self { state, child_storage: Some(child_info), prefix, current_key, _phantom: PhantomData }
}
}
impl<'a, State, Block> Iterator for KeyIterator<'a, State, Block> where
impl<'a, State, Block> Iterator for KeyIterator<'a, State, Block>
where
Block: BlockT,
State: StateBackend<HashFor<Block>>,
{
@@ -327,11 +329,13 @@ impl<'a, State, Block> Iterator for KeyIterator<'a, State, Block> where
self.state.next_child_storage_key(child_info, &self.current_key)
} else {
self.state.next_storage_key(&self.current_key)
}.ok().flatten()?;
}
.ok()
.flatten()?;
// this terminates the iterator the first time it fails.
if let Some(prefix) = self.prefix {
if !next_key.starts_with(&prefix.0[..]) {
return None;
return None
}
}
self.current_key = next_key.clone();
@@ -342,19 +346,31 @@ impl<'a, State, Block> Iterator for KeyIterator<'a, State, Block> where
/// Provides acess to storage primitives
pub trait StorageProvider<Block: BlockT, B: Backend<Block>> {
/// Given a `BlockId` and a key, return the value under the key in that block.
fn storage(&self, id: &BlockId<Block>, key: &StorageKey) -> sp_blockchain::Result<Option<StorageData>>;
fn storage(
&self,
id: &BlockId<Block>,
key: &StorageKey,
) -> sp_blockchain::Result<Option<StorageData>>;
/// Given a `BlockId` and a key prefix, return the matching storage keys in that block.
fn storage_keys(&self, id: &BlockId<Block>, key_prefix: &StorageKey) -> sp_blockchain::Result<Vec<StorageKey>>;
fn storage_keys(
&self,
id: &BlockId<Block>,
key_prefix: &StorageKey,
) -> sp_blockchain::Result<Vec<StorageKey>>;
/// Given a `BlockId` and a key, return the value under the hash in that block.
fn storage_hash(&self, id: &BlockId<Block>, key: &StorageKey) -> sp_blockchain::Result<Option<Block::Hash>>;
fn storage_hash(
&self,
id: &BlockId<Block>,
key: &StorageKey,
) -> sp_blockchain::Result<Option<Block::Hash>>;
/// Given a `BlockId` and a key prefix, return the matching child storage keys and values in that block.
fn storage_pairs(
&self,
id: &BlockId<Block>,
key_prefix: &StorageKey
key_prefix: &StorageKey,
) -> sp_blockchain::Result<Vec<(StorageKey, StorageData)>>;
/// Given a `BlockId` and a key prefix, return a `KeyIterator` iterates matching storage keys in that block.
@@ -362,7 +378,7 @@ pub trait StorageProvider<Block: BlockT, B: Backend<Block>> {
&self,
id: &BlockId<Block>,
prefix: Option<&'a StorageKey>,
start_key: Option<&StorageKey>
start_key: Option<&StorageKey>,
) -> sp_blockchain::Result<KeyIterator<'a, B::State, Block>>;
/// Given a `BlockId`, a key and a child storage key, return the value under the key in that block.
@@ -370,7 +386,7 @@ pub trait StorageProvider<Block: BlockT, B: Backend<Block>> {
&self,
id: &BlockId<Block>,
child_info: &ChildInfo,
key: &StorageKey
key: &StorageKey,
) -> sp_blockchain::Result<Option<StorageData>>;
/// Given a `BlockId`, a key prefix, and a child storage key, return the matching child storage keys.
@@ -378,7 +394,7 @@ pub trait StorageProvider<Block: BlockT, B: Backend<Block>> {
&self,
id: &BlockId<Block>,
child_info: &ChildInfo,
key_prefix: &StorageKey
key_prefix: &StorageKey,
) -> sp_blockchain::Result<Vec<StorageKey>>;
/// Given a `BlockId` and a key `prefix` and a child storage key,
@@ -388,7 +404,7 @@ pub trait StorageProvider<Block: BlockT, B: Backend<Block>> {
id: &BlockId<Block>,
child_info: ChildInfo,
prefix: Option<&'a StorageKey>,
start_key: Option<&StorageKey>
start_key: Option<&StorageKey>,
) -> sp_blockchain::Result<KeyIterator<'a, B::State, Block>>;
/// Given a `BlockId`, a key and a child storage key, return the hash under the key in that block.
@@ -396,7 +412,7 @@ pub trait StorageProvider<Block: BlockT, B: Backend<Block>> {
&self,
id: &BlockId<Block>,
child_info: &ChildInfo,
key: &StorageKey
key: &StorageKey,
) -> sp_blockchain::Result<Option<Block::Hash>>;
/// Get longest range within [first; last] that is possible to use in `key_changes`
@@ -418,7 +434,7 @@ pub trait StorageProvider<Block: BlockT, B: Backend<Block>> {
first: NumberFor<Block>,
last: BlockId<Block>,
storage_key: Option<&PrefixedStorageKey>,
key: &StorageKey
key: &StorageKey,
) -> sp_blockchain::Result<Vec<(NumberFor<Block>, u32)>>;
}
@@ -511,20 +527,20 @@ pub trait Backend<Block: BlockT>: AuxStore + Send + Sync {
) -> sp_blockchain::Result<(NumberFor<Block>, HashSet<Block::Hash>)>;
/// Discard non-best, unfinalized leaf block.
fn remove_leaf_block(
&self,
hash: &Block::Hash,
) -> sp_blockchain::Result<()>;
fn remove_leaf_block(&self, hash: &Block::Hash) -> sp_blockchain::Result<()>;
/// Insert auxiliary data into key-value store.
fn insert_aux<
'a,
'b: 'a,
'c: 'a,
I: IntoIterator<Item=&'a(&'c [u8], &'c [u8])>,
D: IntoIterator<Item=&'a &'b [u8]>,
>(&self, insert: I, delete: D) -> sp_blockchain::Result<()>
{
I: IntoIterator<Item = &'a (&'c [u8], &'c [u8])>,
D: IntoIterator<Item = &'a &'b [u8]>,
>(
&self,
insert: I,
delete: D,
) -> sp_blockchain::Result<()> {
AuxStore::insert_aux(self, insert, delete)
}
/// Query auxiliary data from key-value store.
@@ -548,9 +564,10 @@ pub trait PrunableStateChangesTrieStorage<Block: BlockT>:
/// Get reference to StateChangesTrieStorage.
fn storage(&self) -> &dyn StateChangesTrieStorage<HashFor<Block>, NumberFor<Block>>;
/// Get configuration at given block.
fn configuration_at(&self, at: &BlockId<Block>) -> sp_blockchain::Result<
ChangesTrieConfigurationRange<NumberFor<Block>, Block::Hash>
>;
fn configuration_at(
&self,
at: &BlockId<Block>,
) -> sp_blockchain::Result<ChangesTrieConfigurationRange<NumberFor<Block>, Block::Hash>>;
/// Get end block (inclusive) of oldest pruned max-level (or skewed) digest trie blocks range.
/// It is guaranteed that we have no any changes tries before (and including) this block.
/// It is guaranteed that all existing changes tries after this block are not yet pruned (if created).
@@ -584,7 +601,8 @@ pub fn changes_tries_state_at_block<'a, Block: BlockT>(
let config_range = storage.configuration_at(block)?;
match config_range.config {
Some(config) => Ok(Some(ChangesTrieState::new(config, config_range.zero.0, storage.storage()))),
Some(config) =>
Ok(Some(ChangesTrieState::new(config, config_range.zero.0, storage.storage()))),
None => Ok(None),
}
}
+26 -22
View File
@@ -18,20 +18,19 @@
//! A method call executor interface.
use std::{panic::UnwindSafe, result, cell::RefCell};
use codec::{Encode, Decode};
use sp_runtime::{
generic::BlockId, traits::{Block as BlockT, HashFor},
};
use sp_state_machine::{
OverlayedChanges, ExecutionManager, ExecutionStrategy, StorageProof,
};
use sc_executor::{RuntimeVersion, NativeVersion};
use sp_externalities::Extensions;
use codec::{Decode, Encode};
use sc_executor::{NativeVersion, RuntimeVersion};
use sp_core::NativeOrEncoded;
use sp_externalities::Extensions;
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, HashFor},
};
use sp_state_machine::{ExecutionManager, ExecutionStrategy, OverlayedChanges, StorageProof};
use std::{cell::RefCell, panic::UnwindSafe, result};
use sp_api::{ProofRecorder, StorageTransactionCache};
use crate::execution_extensions::ExecutionExtensions;
use sp_api::{ProofRecorder, StorageTransactionCache};
/// Executor Provider
pub trait ExecutorProvider<Block: BlockT> {
@@ -73,7 +72,7 @@ pub trait CallExecutor<B: BlockT> {
fn contextual_call<
EM: Fn(
Result<NativeOrEncoded<R>, Self::Error>,
Result<NativeOrEncoded<R>, Self::Error>
Result<NativeOrEncoded<R>, Self::Error>,
) -> Result<NativeOrEncoded<R>, Self::Error>,
R: Encode + Decode + PartialEq,
NC: FnOnce() -> result::Result<R, sp_api::ApiError> + UnwindSafe,
@@ -83,14 +82,18 @@ pub trait CallExecutor<B: BlockT> {
method: &str,
call_data: &[u8],
changes: &RefCell<OverlayedChanges>,
storage_transaction_cache: Option<&RefCell<
StorageTransactionCache<B, <Self::Backend as crate::backend::Backend<B>>::State>,
>>,
storage_transaction_cache: Option<
&RefCell<
StorageTransactionCache<B, <Self::Backend as crate::backend::Backend<B>>::State>,
>,
>,
execution_manager: ExecutionManager<EM>,
native_call: Option<NC>,
proof_recorder: &Option<ProofRecorder<B>>,
extensions: Option<Extensions>,
) -> sp_blockchain::Result<NativeOrEncoded<R>> where ExecutionManager<EM>: Clone;
) -> sp_blockchain::Result<NativeOrEncoded<R>>
where
ExecutionManager<EM>: Clone;
/// Extract RuntimeVersion of given block
///
@@ -105,12 +108,13 @@ pub trait CallExecutor<B: BlockT> {
mut state: S,
overlay: &mut OverlayedChanges,
method: &str,
call_data: &[u8]
call_data: &[u8],
) -> Result<(Vec<u8>, StorageProof), sp_blockchain::Error> {
let trie_state = state.as_trie_backend()
.ok_or_else(||
sp_blockchain::Error::from_state(Box::new(sp_state_machine::ExecutionError::UnableToGenerateProof) as Box<_>)
)?;
let trie_state = state.as_trie_backend().ok_or_else(|| {
sp_blockchain::Error::from_state(Box::new(
sp_state_machine::ExecutionError::UnableToGenerateProof,
) as Box<_>)
})?;
self.prove_at_trie_state(trie_state, overlay, method, call_data)
}
@@ -122,7 +126,7 @@ pub trait CallExecutor<B: BlockT> {
trie_state: &sp_state_machine::TrieBackend<S, HashFor<B>>,
overlay: &mut OverlayedChanges,
method: &str,
call_data: &[u8]
call_data: &[u8],
) -> Result<(Vec<u8>, StorageProof), sp_blockchain::Error>;
/// Get runtime version if supported.
+115 -107
View File
@@ -25,15 +25,15 @@
//! root hash. A correct proof implies that the claimed block is identical to the one
//! we discarded.
use hash_db;
use codec::Encode;
use hash_db;
use sp_trie;
use sp_core::{H256, convert_hash};
use sp_runtime::traits::{Header as HeaderT, AtLeast32Bit, Zero, One};
use sp_core::{convert_hash, H256};
use sp_runtime::traits::{AtLeast32Bit, Header as HeaderT, One, Zero};
use sp_state_machine::{
MemoryDB, TrieBackend, Backend as StateBackend, StorageProof, InMemoryBackend,
prove_read_on_trie_backend, read_proof_check, read_proof_check_on_proving_backend
prove_read_on_trie_backend, read_proof_check, read_proof_check_on_proving_backend,
Backend as StateBackend, InMemoryBackend, MemoryDB, StorageProof, TrieBackend,
};
use sp_blockchain::{Error as ClientError, Result as ClientResult};
@@ -49,17 +49,17 @@ pub fn size<N: From<u32>>() -> N {
/// Returns Some(cht_number) if CHT is need to be built when the block with given number is canonized.
pub fn is_build_required<N>(cht_size: N, block_num: N) -> Option<N>
where
N: Clone + AtLeast32Bit,
where
N: Clone + AtLeast32Bit,
{
let block_cht_num = block_to_cht_number(cht_size.clone(), block_num.clone())?;
let two = N::one() + N::one();
if block_cht_num < two {
return None;
return None
}
let cht_start = start_number(cht_size, block_cht_num.clone());
if cht_start != block_num {
return None;
return None
}
Some(block_cht_num - two)
@@ -67,13 +67,13 @@ pub fn is_build_required<N>(cht_size: N, block_num: N) -> Option<N>
/// Returns Some(max_cht_number) if CHT has ever been built given maximal canonical block number.
pub fn max_cht_number<N>(cht_size: N, max_canonical_block: N) -> Option<N>
where
N: Clone + AtLeast32Bit,
where
N: Clone + AtLeast32Bit,
{
let max_cht_number = block_to_cht_number(cht_size, max_canonical_block)?;
let two = N::one() + N::one();
if max_cht_number < two {
return None;
return None
}
Some(max_cht_number - two)
}
@@ -86,16 +86,16 @@ pub fn compute_root<Header, Hasher, I>(
cht_num: Header::Number,
hashes: I,
) -> ClientResult<Hasher::Out>
where
Header: HeaderT,
Hasher: hash_db::Hasher,
Hasher::Out: Ord,
I: IntoIterator<Item=ClientResult<Option<Header::Hash>>>,
where
Header: HeaderT,
Hasher: hash_db::Hasher,
Hasher::Out: Ord,
I: IntoIterator<Item = ClientResult<Option<Header::Hash>>>,
{
use sp_trie::TrieConfiguration;
Ok(sp_trie::trie_types::Layout::<Hasher>::trie_root(
build_pairs::<Header, I>(cht_size, cht_num, hashes)?
))
Ok(sp_trie::trie_types::Layout::<Hasher>::trie_root(build_pairs::<Header, I>(
cht_size, cht_num, hashes,
)?))
}
/// Build CHT-based header proof.
@@ -103,26 +103,28 @@ pub fn build_proof<Header, Hasher, BlocksI, HashesI>(
cht_size: Header::Number,
cht_num: Header::Number,
blocks: BlocksI,
hashes: HashesI
hashes: HashesI,
) -> ClientResult<StorageProof>
where
Header: HeaderT,
Hasher: hash_db::Hasher,
Hasher::Out: Ord + codec::Codec,
BlocksI: IntoIterator<Item=Header::Number>,
HashesI: IntoIterator<Item=ClientResult<Option<Header::Hash>>>,
where
Header: HeaderT,
Hasher: hash_db::Hasher,
Hasher::Out: Ord + codec::Codec,
BlocksI: IntoIterator<Item = Header::Number>,
HashesI: IntoIterator<Item = ClientResult<Option<Header::Hash>>>,
{
let transaction = build_pairs::<Header, _>(cht_size, cht_num, hashes)?
.into_iter()
.map(|(k, v)| (k, Some(v)))
.collect::<Vec<_>>();
let mut storage = InMemoryBackend::<Hasher>::default().update(vec![(None, transaction)]);
let trie_storage = storage.as_trie_backend()
let trie_storage = storage
.as_trie_backend()
.expect("InMemoryState::as_trie_backend always returns Some; qed");
prove_read_on_trie_backend(
trie_storage,
blocks.into_iter().map(|number| encode_cht_key(number)),
).map_err(ClientError::from_state)
)
.map_err(ClientError::from_state)
}
/// Check CHT-based header proof.
@@ -132,25 +134,24 @@ pub fn check_proof<Header, Hasher>(
remote_hash: Header::Hash,
remote_proof: StorageProof,
) -> ClientResult<()>
where
Header: HeaderT,
Hasher: hash_db::Hasher,
Hasher::Out: Ord + codec::Codec,
where
Header: HeaderT,
Hasher: hash_db::Hasher,
Hasher::Out: Ord + codec::Codec,
{
do_check_proof::<Header, Hasher, _>(
local_root,
local_number,
remote_hash,
move |local_root, local_cht_key|
move |local_root, local_cht_key| {
read_proof_check::<Hasher, _>(
local_root,
remote_proof,
::std::iter::once(local_cht_key),
)
.map(|mut map| map
.remove(local_cht_key)
.expect("checked proof of local_cht_key; qed"))
.map_err(ClientError::from_state),
.map(|mut map| map.remove(local_cht_key).expect("checked proof of local_cht_key; qed"))
.map_err(ClientError::from_state)
},
)
}
@@ -161,20 +162,19 @@ pub fn check_proof_on_proving_backend<Header, Hasher>(
remote_hash: Header::Hash,
proving_backend: &TrieBackend<MemoryDB<Hasher>, Hasher>,
) -> ClientResult<()>
where
Header: HeaderT,
Hasher: hash_db::Hasher,
Hasher::Out: Ord + codec::Codec,
where
Header: HeaderT,
Hasher: hash_db::Hasher,
Hasher::Out: Ord + codec::Codec,
{
do_check_proof::<Header, Hasher, _>(
local_root,
local_number,
remote_hash,
|_, local_cht_key|
read_proof_check_on_proving_backend::<Hasher>(
proving_backend,
local_cht_key,
).map_err(ClientError::from_state),
|_, local_cht_key| {
read_proof_check_on_proving_backend::<Hasher>(proving_backend, local_cht_key)
.map_err(ClientError::from_state)
},
)
}
@@ -185,22 +185,22 @@ fn do_check_proof<Header, Hasher, F>(
remote_hash: Header::Hash,
checker: F,
) -> ClientResult<()>
where
Header: HeaderT,
Hasher: hash_db::Hasher,
Hasher::Out: Ord,
F: FnOnce(Hasher::Out, &[u8]) -> ClientResult<Option<Vec<u8>>>,
where
Header: HeaderT,
Hasher: hash_db::Hasher,
Hasher::Out: Ord,
F: FnOnce(Hasher::Out, &[u8]) -> ClientResult<Option<Vec<u8>>>,
{
let root: Hasher::Out = convert_hash(&local_root);
let local_cht_key = encode_cht_key(local_number);
let local_cht_value = checker(root, &local_cht_key)?;
let local_cht_value = local_cht_value.ok_or_else(|| ClientError::InvalidCHTProof)?;
let local_hash = decode_cht_value(&local_cht_value).ok_or_else(|| ClientError::InvalidCHTProof)?;
let local_hash =
decode_cht_value(&local_cht_value).ok_or_else(|| ClientError::InvalidCHTProof)?;
match &local_hash[..] == remote_hash.as_ref() {
true => Ok(()),
false => Err(ClientError::InvalidCHTProof.into()),
}
}
/// Group ordered blocks by CHT number and call functor with blocks of each group.
@@ -210,29 +210,31 @@ pub fn for_each_cht_group<Header, I, F, P>(
mut functor: F,
mut functor_param: P,
) -> ClientResult<()>
where
Header: HeaderT,
I: IntoIterator<Item=Header::Number>,
F: FnMut(P, Header::Number, Vec<Header::Number>) -> ClientResult<P>,
where
Header: HeaderT,
I: IntoIterator<Item = Header::Number>,
F: FnMut(P, Header::Number, Vec<Header::Number>) -> ClientResult<P>,
{
let mut current_cht_num = None;
let mut current_cht_blocks = Vec::new();
for block in blocks {
let new_cht_num = block_to_cht_number(cht_size, block).ok_or_else(|| ClientError::Backend(format!(
"Cannot compute CHT root for the block #{}", block))
)?;
let new_cht_num = block_to_cht_number(cht_size, block).ok_or_else(|| {
ClientError::Backend(format!("Cannot compute CHT root for the block #{}", block))
})?;
let advance_to_next_cht = current_cht_num.is_some() && current_cht_num != Some(new_cht_num);
if advance_to_next_cht {
let current_cht_num = current_cht_num.expect("advance_to_next_cht is true;
it is true only when current_cht_num is Some; qed");
assert!(new_cht_num > current_cht_num, "for_each_cht_group only supports ordered iterators");
let current_cht_num = current_cht_num.expect(
"advance_to_next_cht is true;
it is true only when current_cht_num is Some; qed",
);
assert!(
new_cht_num > current_cht_num,
"for_each_cht_group only supports ordered iterators"
);
functor_param = functor(
functor_param,
current_cht_num,
std::mem::take(&mut current_cht_blocks),
)?;
functor_param =
functor(functor_param, current_cht_num, std::mem::take(&mut current_cht_blocks))?;
}
current_cht_blocks.push(block);
@@ -240,11 +242,7 @@ pub fn for_each_cht_group<Header, I, F, P>(
}
if let Some(current_cht_num) = current_cht_num {
functor(
functor_param,
current_cht_num,
std::mem::take(&mut current_cht_blocks),
)?;
functor(functor_param, current_cht_num, std::mem::take(&mut current_cht_blocks))?;
}
Ok(())
@@ -254,26 +252,22 @@ pub fn for_each_cht_group<Header, I, F, P>(
fn build_pairs<Header, I>(
cht_size: Header::Number,
cht_num: Header::Number,
hashes: I
hashes: I,
) -> ClientResult<Vec<(Vec<u8>, Vec<u8>)>>
where
Header: HeaderT,
I: IntoIterator<Item=ClientResult<Option<Header::Hash>>>,
where
Header: HeaderT,
I: IntoIterator<Item = ClientResult<Option<Header::Hash>>>,
{
let start_num = start_number(cht_size, cht_num);
let mut pairs = Vec::new();
let mut hash_index = Header::Number::zero();
for hash in hashes.into_iter() {
let hash = hash?.ok_or_else(|| ClientError::from(
ClientError::MissingHashRequiredForCHT
))?;
pairs.push((
encode_cht_key(start_num + hash_index).to_vec(),
encode_cht_value(hash)
));
let hash =
hash?.ok_or_else(|| ClientError::from(ClientError::MissingHashRequiredForCHT))?;
pairs.push((encode_cht_key(start_num + hash_index).to_vec(), encode_cht_value(hash)));
hash_index += Header::Number::one();
if hash_index == cht_size {
break;
break
}
}
@@ -325,7 +319,6 @@ pub fn decode_cht_value(value: &[u8]) -> Option<H256> {
32 => Some(H256::from_slice(&value[0..32])),
_ => None,
}
}
#[cfg(test)]
@@ -379,8 +372,12 @@ mod tests {
#[test]
fn build_pairs_fails_when_no_enough_blocks() {
assert!(build_pairs::<Header, _>(SIZE as _, 0,
::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))).take(SIZE as usize / 2)).is_err());
assert!(build_pairs::<Header, _>(
SIZE as _,
0,
::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))).take(SIZE as usize / 2)
)
.is_err());
}
#[test]
@@ -391,9 +388,12 @@ mod tests {
::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1))))
.take(SIZE as usize / 2)
.chain(::std::iter::once(Ok(None)))
.chain(::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(2))))
.take(SIZE as usize / 2 - 1))
).is_err());
.chain(
::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(2))))
.take(SIZE as usize / 2 - 1)
)
)
.is_err());
}
#[test]
@@ -401,9 +401,9 @@ mod tests {
assert!(compute_root::<Header, BlakeTwo256, _>(
SIZE as _,
42,
::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1))))
.take(SIZE as usize)
).is_ok());
::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))).take(SIZE as usize)
)
.is_ok());
}
#[test]
@@ -413,9 +413,9 @@ mod tests {
SIZE as _,
0,
vec![(SIZE * 1000) as u64],
::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1))))
.take(SIZE as usize)
).is_err());
::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))).take(SIZE as usize)
)
.is_err());
}
#[test]
@@ -424,9 +424,9 @@ mod tests {
SIZE as _,
0,
vec![(SIZE / 2) as u64],
::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1))))
.take(SIZE as usize)
).is_ok());
::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))).take(SIZE as usize)
)
.is_ok());
}
#[test]
@@ -447,19 +447,27 @@ mod tests {
let _ = for_each_cht_group::<Header, _, _, _>(
cht_size,
vec![
cht_size * 2 + 1, cht_size * 2 + 2, cht_size * 2 + 5,
cht_size * 4 + 1, cht_size * 4 + 7,
cht_size * 6 + 1
], |_, cht_num, blocks| {
cht_size * 2 + 1,
cht_size * 2 + 2,
cht_size * 2 + 5,
cht_size * 4 + 1,
cht_size * 4 + 7,
cht_size * 6 + 1,
],
|_, cht_num, blocks| {
match cht_num {
2 => assert_eq!(blocks, vec![cht_size * 2 + 1, cht_size * 2 + 2, cht_size * 2 + 5]),
2 => assert_eq!(
blocks,
vec![cht_size * 2 + 1, cht_size * 2 + 2, cht_size * 2 + 5]
),
4 => assert_eq!(blocks, vec![cht_size * 4 + 1, cht_size * 4 + 7]),
6 => assert_eq!(blocks, vec![cht_size * 6 + 1]),
_ => unreachable!(),
}
Ok(())
}, ()
},
(),
);
}
}
+18 -23
View File
@@ -18,20 +18,19 @@
//! A set of APIs supported by the client along with their primitives.
use std::{fmt, collections::HashSet, sync::Arc, convert::TryFrom};
use sp_consensus::BlockOrigin;
use sp_core::storage::StorageKey;
use sp_runtime::{
traits::{Block as BlockT, NumberFor},
generic::{BlockId, SignedBlock},
traits::{Block as BlockT, NumberFor},
Justifications,
};
use sp_consensus::BlockOrigin;
use std::{collections::HashSet, convert::TryFrom, fmt, sync::Arc};
use crate::blockchain::Info;
use crate::notifications::StorageEventStream;
use sp_utils::mpsc::TracingUnboundedReceiver;
use sp_blockchain;
use crate::{blockchain::Info, notifications::StorageEventStream};
use sc_transaction_pool_api::ChainEvent;
use sp_blockchain;
use sp_utils::mpsc::TracingUnboundedReceiver;
/// Type that implements `futures::Stream` of block import events.
pub type ImportNotifications<Block> = TracingUnboundedReceiver<BlockImportNotification<Block>>;
@@ -82,7 +81,7 @@ pub trait BlockBackend<Block: BlockT> {
/// Get block body by ID. Returns `None` if the body is not stored.
fn block_body(
&self,
id: &BlockId<Block>
id: &BlockId<Block>,
) -> sp_blockchain::Result<Option<Vec<<Block as BlockT>::Extrinsic>>>;
/// Get all indexed transactions for a block,
@@ -99,7 +98,8 @@ pub trait BlockBackend<Block: BlockT> {
fn block(&self, id: &BlockId<Block>) -> sp_blockchain::Result<Option<SignedBlock<Block>>>;
/// Get block status.
fn block_status(&self, id: &BlockId<Block>) -> sp_blockchain::Result<sp_consensus::BlockStatus>;
fn block_status(&self, id: &BlockId<Block>)
-> sp_blockchain::Result<sp_consensus::BlockStatus>;
/// Get block justifications for the block with the given id.
fn justifications(&self, id: &BlockId<Block>) -> sp_blockchain::Result<Option<Justifications>>;
@@ -107,14 +107,11 @@ pub trait BlockBackend<Block: BlockT> {
/// Get block hash by number.
fn block_hash(&self, number: NumberFor<Block>) -> sp_blockchain::Result<Option<Block::Hash>>;
/// Get single indexed transaction by content hash.
/// Get single indexed transaction by content hash.
///
/// Note that this will only fetch transactions
/// that are indexed by the runtime with `storage_index_transaction`.
fn indexed_transaction(
&self,
hash: &Block::Hash,
) -> sp_blockchain::Result<Option<Vec<u8>>>;
fn indexed_transaction(&self, hash: &Block::Hash) -> sp_blockchain::Result<Option<Vec<u8>>>;
/// Check if transaction index exists.
fn has_indexed_transaction(&self, hash: &Block::Hash) -> sp_blockchain::Result<bool> {
@@ -125,8 +122,11 @@ pub trait BlockBackend<Block: BlockT> {
/// Provide a list of potential uncle headers for a given block.
pub trait ProvideUncles<Block: BlockT> {
/// Gets the uncles of the block with `target_hash` going back `max_generation` ancestors.
fn uncles(&self, target_hash: Block::Hash, max_generation: NumberFor<Block>)
-> sp_blockchain::Result<Vec<Block::Header>>;
fn uncles(
&self,
target_hash: Block::Hash,
max_generation: NumberFor<Block>,
) -> sp_blockchain::Result<Vec<Block::Header>>;
}
/// Client info
@@ -284,10 +284,7 @@ impl<B: BlockT> TryFrom<BlockImportNotification<B>> for ChainEvent<B> {
fn try_from(n: BlockImportNotification<B>) -> Result<Self, ()> {
if n.is_new_best {
Ok(Self::NewBestBlock {
hash: n.hash,
tree_route: n.tree_route,
})
Ok(Self::NewBestBlock { hash: n.hash, tree_route: n.tree_route })
} else {
Err(())
}
@@ -296,8 +293,6 @@ impl<B: BlockT> TryFrom<BlockImportNotification<B>> for ChainEvent<B> {
impl<B: BlockT> From<FinalityNotification<B>> for ChainEvent<B> {
fn from(n: FinalityNotification<B>) -> Self {
Self::Finalized {
hash: n.hash,
}
Self::Finalized { hash: n.hash }
}
}
@@ -22,22 +22,19 @@
//! strategy for the runtime calls and provide the right `Externalities`
//! extensions to support APIs for particular execution context & capabilities.
use std::sync::{Weak, Arc};
use codec::Decode;
use sp_core::{
ExecutionContext,
offchain::{self, OffchainWorkerExt, TransactionPoolExt, OffchainDbExt},
};
use sp_keystore::{KeystoreExt, SyncCryptoStorePtr};
use sp_runtime::{
generic::BlockId,
traits,
};
use sp_state_machine::{ExecutionManager, DefaultHandler};
pub use sp_state_machine::ExecutionStrategy;
use sp_externalities::Extensions;
use parking_lot::RwLock;
use sc_transaction_pool_api::OffchainSubmitTransaction;
use sp_core::{
offchain::{self, OffchainDbExt, OffchainWorkerExt, TransactionPoolExt},
ExecutionContext,
};
use sp_externalities::Extensions;
use sp_keystore::{KeystoreExt, SyncCryptoStorePtr};
use sp_runtime::{generic::BlockId, traits};
pub use sp_state_machine::ExecutionStrategy;
use sp_state_machine::{DefaultHandler, ExecutionManager};
use std::sync::{Arc, Weak};
/// Execution strategies settings.
#[derive(Debug, Clone)]
@@ -151,7 +148,8 @@ impl<Block: traits::Block> ExecutionExtensions<Block> {
/// Register transaction pool extension.
pub fn register_transaction_pool<T>(&self, pool: &Arc<T>)
where T: OffchainSubmitTransaction<Block> + 'static
where
T: OffchainSubmitTransaction<Block> + 'static,
{
*self.transaction_pool.write() = Some(Arc::downgrade(&pool) as _);
}
@@ -171,14 +169,10 @@ impl<Block: traits::Block> ExecutionExtensions<Block> {
if capabilities.has(offchain::Capability::TransactionPool) {
if let Some(pool) = self.transaction_pool.read().as_ref().and_then(|x| x.upgrade()) {
extensions.register(
TransactionPoolExt(
Box::new(TransactionPoolAdapter {
at: *at,
pool,
}) as _
),
);
extensions
.register(TransactionPoolExt(
Box::new(TransactionPoolAdapter { at: *at, pool }) as _,
));
}
}
@@ -186,19 +180,18 @@ impl<Block: traits::Block> ExecutionExtensions<Block> {
capabilities.has(offchain::Capability::OffchainDbWrite)
{
if let Some(offchain_db) = self.offchain_db.as_ref() {
extensions.register(
OffchainDbExt::new(offchain::LimitedExternalities::new(
capabilities,
offchain_db.create(),
))
);
extensions.register(OffchainDbExt::new(offchain::LimitedExternalities::new(
capabilities,
offchain_db.create(),
)));
}
}
if let ExecutionContext::OffchainCall(Some(ext)) = context {
extensions.register(
OffchainWorkerExt::new(offchain::LimitedExternalities::new(capabilities, ext.0)),
);
extensions.register(OffchainWorkerExt::new(offchain::LimitedExternalities::new(
capabilities,
ext.0,
)));
}
extensions
@@ -212,21 +205,14 @@ impl<Block: traits::Block> ExecutionExtensions<Block> {
&self,
at: &BlockId<Block>,
context: ExecutionContext,
) -> (
ExecutionManager<DefaultHandler<R, E>>,
Extensions,
) {
) -> (ExecutionManager<DefaultHandler<R, E>>, Extensions) {
let manager = match context {
ExecutionContext::BlockConstruction =>
self.strategies.block_construction.get_manager(),
ExecutionContext::Syncing =>
self.strategies.syncing.get_manager(),
ExecutionContext::Importing =>
self.strategies.importing.get_manager(),
ExecutionContext::BlockConstruction => self.strategies.block_construction.get_manager(),
ExecutionContext::Syncing => self.strategies.syncing.get_manager(),
ExecutionContext::Importing => self.strategies.importing.get_manager(),
ExecutionContext::OffchainCall(Some((_, capabilities))) if capabilities.has_all() =>
self.strategies.offchain_worker.get_manager(),
ExecutionContext::OffchainCall(_) =>
self.strategies.other.get_manager(),
ExecutionContext::OffchainCall(_) => self.strategies.other.get_manager(),
};
(manager, self.extensions(at, context))
@@ -245,7 +231,7 @@ impl<Block: traits::Block> offchain::TransactionPool for TransactionPoolAdapter<
Ok(xt) => xt,
Err(e) => {
log::warn!("Unable to decode extrinsic: {:?}: {}", data, e);
return Err(());
return Err(())
},
};
+231 -137
View File
@@ -18,30 +18,31 @@
//! In memory client backend
use std::collections::{HashMap, HashSet};
use std::ptr;
use std::sync::Arc;
use parking_lot::RwLock;
use sp_core::{
storage::well_known_keys, offchain::storage::InMemOffchainStorage as OffchainStorage,
};
use sp_runtime::generic::BlockId;
use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, NumberFor, HashFor};
use sp_runtime::{Justification, Justifications, Storage};
use sp_state_machine::{
ChangesTrieTransaction, InMemoryBackend, Backend as StateBackend, StorageCollection,
ChildStorageCollection, IndexOperation,
};
use sp_blockchain::{CachedHeaderMetadata, HeaderMetadata};
use sp_core::{
offchain::storage::InMemOffchainStorage as OffchainStorage, storage::well_known_keys,
};
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, HashFor, Header as HeaderT, NumberFor, Zero},
Justification, Justifications, Storage,
};
use sp_state_machine::{
Backend as StateBackend, ChangesTrieTransaction, ChildStorageCollection, InMemoryBackend,
IndexOperation, StorageCollection,
};
use std::{
collections::{HashMap, HashSet},
ptr,
sync::Arc,
};
use crate::{
backend::{self, NewBlockState, ProvideChtRoots},
blockchain::{
self, BlockStatus, HeaderBackend, well_known_cache_keys::Id as CacheKeyId
},
UsageInfo,
light,
blockchain::{self, well_known_cache_keys::Id as CacheKeyId, BlockStatus, HeaderBackend},
leaves::LeafSet,
light, UsageInfo,
};
struct PendingBlock<B: BlockT> {
@@ -56,7 +57,11 @@ enum StoredBlock<B: BlockT> {
}
impl<B: BlockT> StoredBlock<B> {
fn new(header: B::Header, body: Option<Vec<B::Extrinsic>>, just: Option<Justifications>) -> Self {
fn new(
header: B::Header,
body: Option<Vec<B::Extrinsic>>,
just: Option<Justifications>,
) -> Self {
match body {
Some(body) => StoredBlock::Full(B::new(header, body), just),
None => StoredBlock::Header(header, just),
@@ -72,7 +77,7 @@ impl<B: BlockT> StoredBlock<B> {
fn justifications(&self) -> Option<&Justifications> {
match *self {
StoredBlock::Header(_, ref j) | StoredBlock::Full(_, ref j) => j.as_ref()
StoredBlock::Header(_, ref j) | StoredBlock::Full(_, ref j) => j.as_ref(),
}
}
@@ -89,7 +94,7 @@ impl<B: BlockT> StoredBlock<B> {
StoredBlock::Full(block, just) => {
let (header, body) = block.deconstruct();
(header, Some(body), just)
}
},
}
}
}
@@ -123,9 +128,7 @@ impl<Block: BlockT> Default for Blockchain<Block> {
impl<Block: BlockT + Clone> Clone for Blockchain<Block> {
fn clone(&self) -> Self {
let storage = Arc::new(RwLock::new(self.storage.read().clone()));
Blockchain {
storage,
}
Blockchain { storage }
}
}
@@ -140,23 +143,20 @@ impl<Block: BlockT> Blockchain<Block> {
/// Create new in-memory blockchain storage.
pub fn new() -> Blockchain<Block> {
let storage = Arc::new(RwLock::new(
BlockchainStorage {
blocks: HashMap::new(),
hashes: HashMap::new(),
best_hash: Default::default(),
best_number: Zero::zero(),
finalized_hash: Default::default(),
finalized_number: Zero::zero(),
genesis_hash: Default::default(),
header_cht_roots: HashMap::new(),
changes_trie_cht_roots: HashMap::new(),
leaves: LeafSet::new(),
aux: HashMap::new(),
}));
Blockchain {
storage,
}
let storage = Arc::new(RwLock::new(BlockchainStorage {
blocks: HashMap::new(),
hashes: HashMap::new(),
best_hash: Default::default(),
best_number: Zero::zero(),
finalized_hash: Default::default(),
finalized_number: Zero::zero(),
genesis_hash: Default::default(),
header_cht_roots: HashMap::new(),
changes_trie_cht_roots: HashMap::new(),
leaves: LeafSet::new(),
aux: HashMap::new(),
}));
Blockchain { storage }
}
/// Insert a block header and associated data.
@@ -175,8 +175,12 @@ impl<Block: BlockT> Blockchain<Block> {
{
let mut storage = self.storage.write();
storage.leaves.import(hash.clone(), number.clone(), header.parent_hash().clone());
storage.blocks.insert(hash.clone(), StoredBlock::new(header, body, justifications));
storage
.leaves
.import(hash.clone(), number.clone(), header.parent_hash().clone());
storage
.blocks
.insert(hash.clone(), StoredBlock::new(header, body, justifications));
if let NewBlockState::Final = new_state {
storage.finalized_hash = hash;
@@ -200,7 +204,7 @@ impl<Block: BlockT> Blockchain<Block> {
pub fn equals_to(&self, other: &Self) -> bool {
// Check ptr equality first to avoid double read locks.
if ptr::eq(self, other) {
return true;
return true
}
self.canon_equals_to(other) && self.storage.read().blocks == other.storage.read().blocks
}
@@ -209,14 +213,14 @@ impl<Block: BlockT> Blockchain<Block> {
pub fn canon_equals_to(&self, other: &Self) -> bool {
// Check ptr equality first to avoid double read locks.
if ptr::eq(self, other) {
return true;
return true
}
let this = self.storage.read();
let other = other.storage.read();
this.hashes == other.hashes
&& this.best_hash == other.best_hash
&& this.best_number == other.best_number
&& this.genesis_hash == other.genesis_hash
this.hashes == other.hashes &&
this.best_hash == other.best_hash &&
this.best_number == other.best_number &&
this.genesis_hash == other.genesis_hash
}
/// Insert header CHT root.
@@ -226,7 +230,8 @@ impl<Block: BlockT> Blockchain<Block> {
/// Set an existing block as head.
pub fn set_head(&self, id: BlockId<Block>) -> sp_blockchain::Result<()> {
let header = self.header(id)?
let header = self
.header(id)?
.ok_or_else(|| sp_blockchain::Error::UnknownBlock(format!("{}", id)))?;
self.apply_head(&header)
@@ -270,7 +275,11 @@ impl<Block: BlockT> Blockchain<Block> {
Ok(())
}
fn finalize_header(&self, id: BlockId<Block>, justification: Option<Justification>) -> sp_blockchain::Result<()> {
fn finalize_header(
&self,
id: BlockId<Block>,
justification: Option<Justification>,
) -> sp_blockchain::Result<()> {
let hash = match self.header(id)? {
Some(h) => h.hash(),
None => return Err(sp_blockchain::Error::UnknownBlock(format!("{}", id))),
@@ -280,11 +289,13 @@ impl<Block: BlockT> Blockchain<Block> {
storage.finalized_hash = hash;
if justification.is_some() {
let block = storage.blocks.get_mut(&hash)
let block = storage
.blocks
.get_mut(&hash)
.expect("hash was fetched from a block in the db; qed");
let block_justifications = match block {
StoredBlock::Header(_, ref mut j) | StoredBlock::Full(_, ref mut j) => j
StoredBlock::Header(_, ref mut j) | StoredBlock::Full(_, ref mut j) => j,
};
*block_justifications = justification.map(Justifications::from);
@@ -293,9 +304,11 @@ impl<Block: BlockT> Blockchain<Block> {
Ok(())
}
fn append_justification(&self, id: BlockId<Block>, justification: Justification)
-> sp_blockchain::Result<()>
{
fn append_justification(
&self,
id: BlockId<Block>,
justification: Justification,
) -> sp_blockchain::Result<()> {
let hash = self.expect_block_hash_from_id(&id)?;
let mut storage = self.storage.write();
@@ -305,14 +318,14 @@ impl<Block: BlockT> Blockchain<Block> {
.expect("hash was fetched from a block in the db; qed");
let block_justifications = match block {
StoredBlock::Header(_, ref mut j) | StoredBlock::Full(_, ref mut j) => j
StoredBlock::Header(_, ref mut j) | StoredBlock::Full(_, ref mut j) => j,
};
if let Some(stored_justifications) = block_justifications {
if !stored_justifications.append(justification) {
return Err(sp_blockchain::Error::BadJustification(
"Duplicate consensus engine ID".into()
));
"Duplicate consensus engine ID".into(),
))
}
} else {
*block_justifications = Some(Justifications::from(justification));
@@ -333,10 +346,13 @@ impl<Block: BlockT> Blockchain<Block> {
}
impl<Block: BlockT> HeaderBackend<Block> for Blockchain<Block> {
fn header(&self, id: BlockId<Block>) -> sp_blockchain::Result<Option<<Block as BlockT>::Header>> {
Ok(self.id(id).and_then(|hash| {
self.storage.read().blocks.get(&hash).map(|b| b.header().clone())
}))
fn header(
&self,
id: BlockId<Block>,
) -> sp_blockchain::Result<Option<<Block as BlockT>::Header>> {
Ok(self
.id(id)
.and_then(|hash| self.storage.read().blocks.get(&hash).map(|b| b.header().clone())))
}
fn info(&self) -> blockchain::Info<Block> {
@@ -352,7 +368,7 @@ impl<Block: BlockT> HeaderBackend<Block> for Blockchain<Block> {
} else {
None
},
number_leaves: storage.leaves.count()
number_leaves: storage.leaves.count(),
}
}
@@ -367,7 +383,10 @@ impl<Block: BlockT> HeaderBackend<Block> for Blockchain<Block> {
Ok(self.storage.read().blocks.get(&hash).map(|b| *b.header().number()))
}
fn hash(&self, number: <<Block as BlockT>::Header as HeaderT>::Number) -> sp_blockchain::Result<Option<Block::Hash>> {
fn hash(
&self,
number: <<Block as BlockT>::Header as HeaderT>::Number,
) -> sp_blockchain::Result<Option<Block::Hash>> {
Ok(self.id(BlockId::Number(number)))
}
}
@@ -375,9 +394,15 @@ impl<Block: BlockT> HeaderBackend<Block> for Blockchain<Block> {
impl<Block: BlockT> HeaderMetadata<Block> for Blockchain<Block> {
type Error = sp_blockchain::Error;
fn header_metadata(&self, hash: Block::Hash) -> Result<CachedHeaderMetadata<Block>, Self::Error> {
self.header(BlockId::hash(hash))?.map(|header| CachedHeaderMetadata::from(&header))
.ok_or_else(|| sp_blockchain::Error::UnknownBlock(format!("header not found: {}", hash)))
fn header_metadata(
&self,
hash: Block::Hash,
) -> Result<CachedHeaderMetadata<Block>, Self::Error> {
self.header(BlockId::hash(hash))?
.map(|header| CachedHeaderMetadata::from(&header))
.ok_or_else(|| {
sp_blockchain::Error::UnknownBlock(format!("header not found: {}", hash))
})
}
fn insert_header_metadata(&self, _hash: Block::Hash, _metadata: CachedHeaderMetadata<Block>) {
@@ -389,17 +414,27 @@ impl<Block: BlockT> HeaderMetadata<Block> for Blockchain<Block> {
}
impl<Block: BlockT> blockchain::Backend<Block> for Blockchain<Block> {
fn body(&self, id: BlockId<Block>) -> sp_blockchain::Result<Option<Vec<<Block as BlockT>::Extrinsic>>> {
fn body(
&self,
id: BlockId<Block>,
) -> sp_blockchain::Result<Option<Vec<<Block as BlockT>::Extrinsic>>> {
Ok(self.id(id).and_then(|hash| {
self.storage.read().blocks.get(&hash)
self.storage
.read()
.blocks
.get(&hash)
.and_then(|b| b.extrinsics().map(|x| x.to_vec()))
}))
}
fn justifications(&self, id: BlockId<Block>) -> sp_blockchain::Result<Option<Justifications>> {
Ok(self.id(id).and_then(|hash| self.storage.read().blocks.get(&hash).and_then(|b|
b.justifications().map(|x| x.clone()))
))
Ok(self.id(id).and_then(|hash| {
self.storage
.read()
.blocks
.get(&hash)
.and_then(|b| b.justifications().map(|x| x.clone()))
}))
}
fn last_finalized(&self) -> sp_blockchain::Result<Block::Hash> {
@@ -418,16 +453,13 @@ impl<Block: BlockT> blockchain::Backend<Block> for Blockchain<Block> {
unimplemented!()
}
fn indexed_transaction(
&self,
_hash: &Block::Hash,
) -> sp_blockchain::Result<Option<Vec<u8>>> {
fn indexed_transaction(&self, _hash: &Block::Hash) -> sp_blockchain::Result<Option<Vec<u8>>> {
unimplemented!("Not supported by the in-mem backend.")
}
fn block_indexed_body(
&self,
_id: BlockId<Block>
_id: BlockId<Block>,
) -> sp_blockchain::Result<Option<Vec<Vec<u8>>>> {
unimplemented!("Not supported by the in-mem backend.")
}
@@ -444,9 +476,13 @@ impl<Block: BlockT> backend::AuxStore for Blockchain<Block> {
'a,
'b: 'a,
'c: 'a,
I: IntoIterator<Item=&'a(&'c [u8], &'c [u8])>,
D: IntoIterator<Item=&'a &'b [u8]>,
>(&self, insert: I, delete: D) -> sp_blockchain::Result<()> {
I: IntoIterator<Item = &'a (&'c [u8], &'c [u8])>,
D: IntoIterator<Item = &'a &'b [u8]>,
>(
&self,
insert: I,
delete: D,
) -> sp_blockchain::Result<()> {
let mut storage = self.storage.write();
for (k, v) in insert {
storage.aux.insert(k.to_vec(), v.to_vec());
@@ -463,8 +499,8 @@ impl<Block: BlockT> backend::AuxStore for Blockchain<Block> {
}
impl<Block: BlockT> light::Storage<Block> for Blockchain<Block>
where
Block::Hash: From<[u8; 32]>,
where
Block::Hash: From<[u8; 32]>,
{
fn import_header(
&self,
@@ -507,8 +543,14 @@ impl<Block: BlockT> ProvideChtRoots<Block> for Blockchain<Block> {
_cht_size: NumberFor<Block>,
block: NumberFor<Block>,
) -> sp_blockchain::Result<Option<Block::Hash>> {
self.storage.read().header_cht_roots.get(&block).cloned()
.ok_or_else(|| sp_blockchain::Error::Backend(format!("Header CHT for block {} not exists", block)))
self.storage
.read()
.header_cht_roots
.get(&block)
.cloned()
.ok_or_else(|| {
sp_blockchain::Error::Backend(format!("Header CHT for block {} not exists", block))
})
.map(Some)
}
@@ -517,8 +559,17 @@ impl<Block: BlockT> ProvideChtRoots<Block> for Blockchain<Block> {
_cht_size: NumberFor<Block>,
block: NumberFor<Block>,
) -> sp_blockchain::Result<Option<Block::Hash>> {
self.storage.read().changes_trie_cht_roots.get(&block).cloned()
.ok_or_else(|| sp_blockchain::Error::Backend(format!("Changes trie CHT for block {} not exists", block)))
self.storage
.read()
.changes_trie_cht_roots
.get(&block)
.cloned()
.ok_or_else(|| {
sp_blockchain::Error::Backend(format!(
"Changes trie CHT for block {} not exists",
block
))
})
.map(Some)
}
}
@@ -527,25 +578,30 @@ impl<Block: BlockT> ProvideChtRoots<Block> for Blockchain<Block> {
pub struct BlockImportOperation<Block: BlockT> {
pending_block: Option<PendingBlock<Block>>,
old_state: InMemoryBackend<HashFor<Block>>,
new_state: Option<<InMemoryBackend<HashFor<Block>> as StateBackend<HashFor<Block>>>::Transaction>,
new_state:
Option<<InMemoryBackend<HashFor<Block>> as StateBackend<HashFor<Block>>>::Transaction>,
aux: Vec<(Vec<u8>, Option<Vec<u8>>)>,
finalized_blocks: Vec<(BlockId<Block>, Option<Justification>)>,
set_head: Option<BlockId<Block>>,
}
impl<Block: BlockT> BlockImportOperation<Block> where
impl<Block: BlockT> BlockImportOperation<Block>
where
Block::Hash: Ord,
{
fn apply_storage(&mut self, storage: Storage, commit: bool) -> sp_blockchain::Result<Block::Hash> {
fn apply_storage(
&mut self,
storage: Storage,
commit: bool,
) -> sp_blockchain::Result<Block::Hash> {
check_genesis_storage(&storage)?;
let child_delta = storage.children_default.iter()
.map(|(_storage_key, child_content)|
(
&child_content.child_info,
child_content.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref())))
)
);
let child_delta = storage.children_default.iter().map(|(_storage_key, child_content)| {
(
&child_content.child_info,
child_content.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))),
)
});
let (root, transaction) = self.old_state.full_storage_root(
storage.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))),
@@ -559,7 +615,8 @@ impl<Block: BlockT> BlockImportOperation<Block> where
}
}
impl<Block: BlockT> backend::BlockImportOperation<Block> for BlockImportOperation<Block> where
impl<Block: BlockT> backend::BlockImportOperation<Block> for BlockImportOperation<Block>
where
Block::Hash: Ord,
{
type State = InMemoryBackend<HashFor<Block>>;
@@ -577,10 +634,8 @@ impl<Block: BlockT> backend::BlockImportOperation<Block> for BlockImportOperatio
state: NewBlockState,
) -> sp_blockchain::Result<()> {
assert!(self.pending_block.is_none(), "Only one block per operation is allowed");
self.pending_block = Some(PendingBlock {
block: StoredBlock::new(header, body, justifications),
state,
});
self.pending_block =
Some(PendingBlock { block: StoredBlock::new(header, body, justifications), state });
Ok(())
}
@@ -601,7 +656,11 @@ impl<Block: BlockT> backend::BlockImportOperation<Block> for BlockImportOperatio
Ok(())
}
fn set_genesis_state(&mut self, storage: Storage, commit: bool) -> sp_blockchain::Result<Block::Hash> {
fn set_genesis_state(
&mut self,
storage: Storage,
commit: bool,
) -> sp_blockchain::Result<Block::Hash> {
self.apply_storage(storage, commit)
}
@@ -610,7 +669,8 @@ impl<Block: BlockT> backend::BlockImportOperation<Block> for BlockImportOperatio
}
fn insert_aux<I>(&mut self, ops: I) -> sp_blockchain::Result<()>
where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
where
I: IntoIterator<Item = (Vec<u8>, Option<Vec<u8>>)>,
{
self.aux.append(&mut ops.into_iter().collect());
Ok(())
@@ -639,7 +699,10 @@ impl<Block: BlockT> backend::BlockImportOperation<Block> for BlockImportOperatio
Ok(())
}
fn update_transaction_index(&mut self, _index: Vec<IndexOperation>) -> sp_blockchain::Result<()> {
fn update_transaction_index(
&mut self,
_index: Vec<IndexOperation>,
) -> sp_blockchain::Result<()> {
Ok(())
}
}
@@ -648,13 +711,19 @@ impl<Block: BlockT> backend::BlockImportOperation<Block> for BlockImportOperatio
///
/// > **Warning**: Doesn't support all the features necessary for a proper database. Only use this
/// > struct for testing purposes. Do **NOT** use in production.
pub struct Backend<Block: BlockT> where Block::Hash: Ord {
pub struct Backend<Block: BlockT>
where
Block::Hash: Ord,
{
states: RwLock<HashMap<Block::Hash, InMemoryBackend<HashFor<Block>>>>,
blockchain: Blockchain<Block>,
import_lock: RwLock<()>,
}
impl<Block: BlockT> Backend<Block> where Block::Hash: Ord {
impl<Block: BlockT> Backend<Block>
where
Block::Hash: Ord,
{
/// Create a new instance of in-mem backend.
pub fn new() -> Self {
Backend {
@@ -665,14 +734,21 @@ impl<Block: BlockT> Backend<Block> where Block::Hash: Ord {
}
}
impl<Block: BlockT> backend::AuxStore for Backend<Block> where Block::Hash: Ord {
impl<Block: BlockT> backend::AuxStore for Backend<Block>
where
Block::Hash: Ord,
{
fn insert_aux<
'a,
'b: 'a,
'c: 'a,
I: IntoIterator<Item=&'a(&'c [u8], &'c [u8])>,
D: IntoIterator<Item=&'a &'b [u8]>,
>(&self, insert: I, delete: D) -> sp_blockchain::Result<()> {
I: IntoIterator<Item = &'a (&'c [u8], &'c [u8])>,
D: IntoIterator<Item = &'a &'b [u8]>,
>(
&self,
insert: I,
delete: D,
) -> sp_blockchain::Result<()> {
self.blockchain.insert_aux(insert, delete)
}
@@ -681,7 +757,10 @@ impl<Block: BlockT> backend::AuxStore for Backend<Block> where Block::Hash: Ord
}
}
impl<Block: BlockT> backend::Backend<Block> for Backend<Block> where Block::Hash: Ord {
impl<Block: BlockT> backend::Backend<Block> for Backend<Block>
where
Block::Hash: Ord,
{
type BlockImportOperation = BlockImportOperation<Block>;
type Blockchain = Blockchain<Block>;
type State = InMemoryBackend<HashFor<Block>>;
@@ -708,10 +787,7 @@ impl<Block: BlockT> backend::Backend<Block> for Backend<Block> where Block::Hash
Ok(())
}
fn commit_operation(
&self,
operation: Self::BlockImportOperation,
) -> sp_blockchain::Result<()> {
fn commit_operation(&self, operation: Self::BlockImportOperation) -> sp_blockchain::Result<()> {
if !operation.finalized_blocks.is_empty() {
for (block, justification) in operation.finalized_blocks {
self.blockchain.finalize_header(block, justification)?;
@@ -779,13 +855,13 @@ impl<Block: BlockT> backend::Backend<Block> for Backend<Block> where Block::Hash
fn state_at(&self, block: BlockId<Block>) -> sp_blockchain::Result<Self::State> {
match block {
BlockId::Hash(h) if h == Default::default() => {
return Ok(Self::State::default());
},
BlockId::Hash(h) if h == Default::default() => return Ok(Self::State::default()),
_ => {},
}
self.blockchain.id(block).and_then(|id| self.states.read().get(&id).cloned())
self.blockchain
.id(block)
.and_then(|id| self.states.read().get(&id).cloned())
.ok_or_else(|| sp_blockchain::Error::UnknownBlock(format!("{}", block)))
}
@@ -797,10 +873,7 @@ impl<Block: BlockT> backend::Backend<Block> for Backend<Block> where Block::Hash
Ok((Zero::zero(), HashSet::new()))
}
fn remove_leaf_block(
&self,
_hash: &Block::Hash,
) -> sp_blockchain::Result<()> {
fn remove_leaf_block(&self, _hash: &Block::Hash) -> sp_blockchain::Result<()> {
Ok(())
}
@@ -811,9 +884,13 @@ impl<Block: BlockT> backend::Backend<Block> for Backend<Block> where Block::Hash
impl<Block: BlockT> backend::LocalBackend<Block> for Backend<Block> where Block::Hash: Ord {}
impl<Block: BlockT> backend::RemoteBackend<Block> for Backend<Block> where Block::Hash: Ord {
impl<Block: BlockT> backend::RemoteBackend<Block> for Backend<Block>
where
Block::Hash: Ord,
{
fn is_local_state_available(&self, block: &BlockId<Block>) -> bool {
self.blockchain.expect_block_number_from_id(block)
self.blockchain
.expect_block_number_from_id(block)
.map(|num| num.is_zero())
.unwrap_or(false)
}
@@ -826,12 +903,15 @@ impl<Block: BlockT> backend::RemoteBackend<Block> for Backend<Block> where Block
/// Check that genesis storage is valid.
pub fn check_genesis_storage(storage: &Storage) -> sp_blockchain::Result<()> {
if storage.top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) {
return Err(sp_blockchain::Error::InvalidState.into());
return Err(sp_blockchain::Error::InvalidState.into())
}
if storage.children_default.keys()
.any(|child_key| !well_known_keys::is_child_storage_key(&child_key)) {
return Err(sp_blockchain::Error::InvalidState.into());
if storage
.children_default
.keys()
.any(|child_key| !well_known_keys::is_child_storage_key(&child_key))
{
return Err(sp_blockchain::Error::InvalidState.into())
}
Ok(())
@@ -839,10 +919,10 @@ pub fn check_genesis_storage(storage: &Storage) -> sp_blockchain::Result<()> {
#[cfg(test)]
mod tests {
use crate::{NewBlockState, in_mem::Blockchain};
use crate::{in_mem::Blockchain, NewBlockState};
use sp_api::{BlockId, HeaderT};
use sp_runtime::{ConsensusEngineId, Justifications};
use sp_blockchain::Backend;
use sp_runtime::{ConsensusEngineId, Justifications};
use substrate_test_runtime::{Block, Header, H256};
pub const ID1: ConsensusEngineId = *b"TST1";
@@ -853,7 +933,13 @@ mod tests {
0 => Default::default(),
_ => header(number - 1).hash(),
};
Header::new(number, H256::from_low_u64_be(0), H256::from_low_u64_be(0), parent_hash, Default::default())
Header::new(
number,
H256::from_low_u64_be(0),
H256::from_low_u64_be(0),
parent_hash,
Default::default(),
)
}
fn test_blockchain() -> Blockchain<Block> {
@@ -862,10 +948,18 @@ mod tests {
let just1 = Some(Justifications::from((ID1, vec![1])));
let just2 = None;
let just3 = Some(Justifications::from((ID1, vec![3])));
blockchain.insert(header(0).hash(), header(0), just0, None, NewBlockState::Final).unwrap();
blockchain.insert(header(1).hash(), header(1), just1, None, NewBlockState::Final).unwrap();
blockchain.insert(header(2).hash(), header(2), just2, None, NewBlockState::Best).unwrap();
blockchain.insert(header(3).hash(), header(3), just3, None, NewBlockState::Final).unwrap();
blockchain
.insert(header(0).hash(), header(0), just0, None, NewBlockState::Final)
.unwrap();
blockchain
.insert(header(1).hash(), header(1), just1, None, NewBlockState::Final)
.unwrap();
blockchain
.insert(header(2).hash(), header(2), just2, None, NewBlockState::Best)
.unwrap();
blockchain
.insert(header(3).hash(), header(3), just3, None, NewBlockState::Final)
.unwrap();
blockchain
}
+41 -38
View File
@@ -18,12 +18,11 @@
//! Helper for managing the set of available leaves in the chain for DB implementations.
use std::collections::BTreeMap;
use std::cmp::Reverse;
use codec::{Decode, Encode};
use sp_blockchain::{Error, Result};
use sp_database::{Database, Transaction};
use sp_runtime::traits::AtLeast32Bit;
use codec::{Encode, Decode};
use sp_blockchain::{Error, Result};
use std::{cmp::Reverse, collections::BTreeMap};
type DbHash = sp_core::H256;
@@ -57,7 +56,7 @@ impl<H, N: Ord> FinalizationDisplaced<H, N> {
}
/// Iterate over all displaced leaves.
pub fn leaves(&self) -> impl IntoIterator<Item=&H> {
pub fn leaves(&self) -> impl IntoIterator<Item = &H> {
self.leaves.values().flatten()
}
}
@@ -72,17 +71,14 @@ pub struct LeafSet<H, N> {
pending_removed: Vec<H>,
}
impl<H, N> LeafSet<H, N> where
impl<H, N> LeafSet<H, N>
where
H: Clone + PartialEq + Decode + Encode,
N: std::fmt::Debug + Clone + AtLeast32Bit + Decode + Encode,
{
/// Construct a new, blank leaf set.
pub fn new() -> Self {
Self {
storage: BTreeMap::new(),
pending_added: Vec::new(),
pending_removed: Vec::new(),
}
Self { storage: BTreeMap::new(), pending_added: Vec::new(), pending_removed: Vec::new() }
}
/// Read the leaf list from the DB, using given prefix for keys.
@@ -98,14 +94,10 @@ impl<H, N> LeafSet<H, N> where
for (number, hashes) in vals.into_iter() {
storage.insert(Reverse(number), hashes);
}
}
},
None => {},
}
Ok(Self {
storage,
pending_added: Vec::new(),
pending_removed: Vec::new(),
})
Ok(Self { storage, pending_added: Vec::new(), pending_removed: Vec::new() })
}
/// update the leaf list on import. returns a displaced leaf if there was one.
@@ -119,10 +111,7 @@ impl<H, N> LeafSet<H, N> where
self.pending_removed.push(parent_hash.clone());
Some(ImportDisplaced {
new_hash: hash.clone(),
displaced: LeafSetItem {
hash: parent_hash,
number: new_number,
},
displaced: LeafSetItem { hash: parent_hash, number: new_number },
})
} else {
None
@@ -144,16 +133,15 @@ impl<H, N> LeafSet<H, N> where
/// will be pruned soon afterwards anyway.
pub fn finalize_height(&mut self, number: N) -> FinalizationDisplaced<H, N> {
let boundary = if number == N::zero() {
return FinalizationDisplaced { leaves: BTreeMap::new() };
return FinalizationDisplaced { leaves: BTreeMap::new() }
} else {
number - N::one()
};
let below_boundary = self.storage.split_off(&Reverse(boundary));
self.pending_removed.extend(below_boundary.values().flat_map(|h| h.iter()).cloned());
FinalizationDisplaced {
leaves: below_boundary,
}
self.pending_removed
.extend(below_boundary.values().flat_map(|h| h.iter()).cloned());
FinalizationDisplaced { leaves: below_boundary }
}
/// Undo all pending operations.
@@ -169,7 +157,9 @@ impl<H, N> LeafSet<H, N> where
/// Revert to the given block height by dropping all leaves in the leaf set
/// with a block number higher than the target.
pub fn revert(&mut self, best_hash: H, best_number: N) {
let items = self.storage.iter()
let items = self
.storage
.iter()
.flat_map(|(number, hashes)| hashes.iter().map(move |h| (h.clone(), number.clone())))
.collect::<Vec<_>>();
@@ -185,7 +175,8 @@ impl<H, N> LeafSet<H, N> where
}
let best_number = Reverse(best_number);
let leaves_contains_best = self.storage
let leaves_contains_best = self
.storage
.get(&best_number)
.map_or(false, |hashes| hashes.contains(&best_hash));
@@ -209,7 +200,12 @@ impl<H, N> LeafSet<H, N> where
}
/// Write the leaf list to the database transaction.
pub fn prepare_transaction(&mut self, tx: &mut Transaction<DbHash>, column: u32, prefix: &[u8]) {
pub fn prepare_transaction(
&mut self,
tx: &mut Transaction<DbHash>,
column: u32,
prefix: &[u8],
) {
let leaves: Vec<_> = self.storage.iter().map(|(n, h)| (n.0.clone(), h.clone())).collect();
tx.set_from_vec(column, prefix, leaves.encode());
self.pending_added.clear();
@@ -218,7 +214,9 @@ impl<H, N> LeafSet<H, N> where
/// Check if given block is a leaf.
pub fn contains(&self, number: N, hash: H) -> bool {
self.storage.get(&Reverse(number)).map_or(false, |hashes| hashes.contains(&hash))
self.storage
.get(&Reverse(number))
.map_or(false, |hashes| hashes.contains(&hash))
}
fn insert_leaf(&mut self, number: Reverse<N>, hash: H) {
@@ -230,14 +228,18 @@ impl<H, N> LeafSet<H, N> where
let mut empty = false;
let removed = self.storage.get_mut(number).map_or(false, |leaves| {
let mut found = false;
leaves.retain(|h| if h == hash {
found = true;
false
} else {
true
leaves.retain(|h| {
if h == hash {
found = true;
false
} else {
true
}
});
if leaves.is_empty() { empty = true }
if leaves.is_empty() {
empty = true
}
found
});
@@ -255,7 +257,8 @@ pub struct Undo<'a, H: 'a, N: 'a> {
inner: &'a mut LeafSet<H, N>,
}
impl<'a, H: 'a, N: 'a> Undo<'a, H, N> where
impl<'a, H: 'a, N: 'a> Undo<'a, H, N>
where
H: Clone + PartialEq + Decode + Encode,
N: std::fmt::Debug + Clone + AtLeast32Bit + Decode + Encode,
{
@@ -329,7 +332,7 @@ mod tests {
fn two_leaves_same_height_can_be_included() {
let mut set = LeafSet::new();
set.import(1_1u32, 10u32,0u32);
set.import(1_1u32, 10u32, 0u32);
set.import(1_2, 10, 0);
assert!(set.storage.contains_key(&Reverse(10)));
+15 -12
View File
@@ -21,30 +21,28 @@
pub mod backend;
pub mod call_executor;
pub mod client;
pub mod cht;
pub mod client;
pub mod execution_extensions;
pub mod in_mem;
pub mod light;
pub mod leaves;
pub mod light;
pub mod notifications;
pub mod proof_provider;
pub use sp_blockchain as blockchain;
pub use backend::*;
pub use notifications::*;
pub use call_executor::*;
pub use client::*;
pub use light::*;
pub use notifications::*;
pub use proof_provider::*;
pub use sp_blockchain as blockchain;
pub use sp_blockchain::HeaderBackend;
pub use sp_state_machine::{StorageProof, ExecutionStrategy};
pub use sp_storage::{StorageData, StorageKey, PrefixedStorageKey, ChildInfo};
pub use sp_state_machine::{ExecutionStrategy, StorageProof};
pub use sp_storage::{ChildInfo, PrefixedStorageKey, StorageData, StorageKey};
/// Usage Information Provider interface
///
pub trait UsageProvider<Block: sp_runtime::traits::Block> {
/// Get usage info about current client.
fn usage_info(&self) -> ClientInfo<Block>;
@@ -52,7 +50,7 @@ pub trait UsageProvider<Block: sp_runtime::traits::Block> {
/// Utility methods for the client.
pub mod utils {
use sp_blockchain::{HeaderBackend, HeaderMetadata, Error};
use sp_blockchain::{Error, HeaderBackend, HeaderMetadata};
use sp_runtime::traits::Block as BlockT;
use std::borrow::Borrow;
@@ -66,19 +64,24 @@ pub mod utils {
client: &'a T,
current: Option<(Block::Hash, Block::Hash)>,
) -> impl Fn(&Block::Hash, &Block::Hash) -> Result<bool, Error> + 'a
where T: HeaderBackend<Block> + HeaderMetadata<Block, Error = Error>,
where
T: HeaderBackend<Block> + HeaderMetadata<Block, Error = Error>,
{
move |base, hash| {
if base == hash { return Ok(false); }
if base == hash {
return Ok(false)
}
let current = current.as_ref().map(|(c, p)| (c.borrow(), p.borrow()));
let mut hash = hash;
if let Some((current_hash, current_parent_hash)) = current {
if base == current_hash { return Ok(false); }
if base == current_hash {
return Ok(false)
}
if hash == current_hash {
if base == current_parent_hash {
return Ok(true);
return Ok(true)
} else {
hash = current_parent_hash;
}
+69 -62
View File
@@ -18,23 +18,26 @@
//! Substrate light client interfaces
use std::sync::Arc;
use std::collections::{BTreeMap, HashMap};
use std::future::Future;
use std::{
collections::{BTreeMap, HashMap},
future::Future,
sync::Arc,
};
use sp_runtime::{
traits::{
Block as BlockT, Header as HeaderT, NumberFor,
},
generic::BlockId
use crate::{
backend::{AuxStore, NewBlockState},
ProvideChtRoots, UsageInfo,
};
use sp_core::{ChangesTrieConfigurationRange, storage::PrefixedStorageKey};
use sp_state_machine::StorageProof;
use sp_blockchain::{
HeaderMetadata, well_known_cache_keys, HeaderBackend, Cache as BlockchainCache,
Error as ClientError, Result as ClientResult,
well_known_cache_keys, Cache as BlockchainCache, Error as ClientError, HeaderBackend,
HeaderMetadata, Result as ClientResult,
};
use crate::{backend::{AuxStore, NewBlockState}, UsageInfo, ProvideChtRoots};
use sp_core::{storage::PrefixedStorageKey, ChangesTrieConfigurationRange};
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, Header as HeaderT, NumberFor},
};
use sp_state_machine::StorageProof;
/// Remote call request.
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
@@ -142,48 +145,48 @@ pub struct RemoteBodyRequest<Header: HeaderT> {
/// is correct (see FetchedDataChecker) and return already checked data.
pub trait Fetcher<Block: BlockT>: Send + Sync {
/// Remote header future.
type RemoteHeaderResult: Future<Output = Result<
Block::Header,
ClientError,
>> + Unpin + Send + 'static;
type RemoteHeaderResult: Future<Output = Result<Block::Header, ClientError>>
+ Unpin
+ Send
+ 'static;
/// Remote storage read future.
type RemoteReadResult: Future<Output = Result<
HashMap<Vec<u8>, Option<Vec<u8>>>,
ClientError,
>> + Unpin + Send + 'static;
type RemoteReadResult: Future<Output = Result<HashMap<Vec<u8>, Option<Vec<u8>>>, ClientError>>
+ Unpin
+ Send
+ 'static;
/// Remote call result future.
type RemoteCallResult: Future<Output = Result<
Vec<u8>,
ClientError,
>> + Unpin + Send + 'static;
type RemoteCallResult: Future<Output = Result<Vec<u8>, ClientError>> + Unpin + Send + 'static;
/// Remote changes result future.
type RemoteChangesResult: Future<Output = Result<
Vec<(NumberFor<Block>, u32)>,
ClientError,
>> + Unpin + Send + 'static;
type RemoteChangesResult: Future<Output = Result<Vec<(NumberFor<Block>, u32)>, ClientError>>
+ Unpin
+ Send
+ 'static;
/// Remote block body result future.
type RemoteBodyResult: Future<Output = Result<
Vec<Block::Extrinsic>,
ClientError,
>> + Unpin + Send + 'static;
type RemoteBodyResult: Future<Output = Result<Vec<Block::Extrinsic>, ClientError>>
+ Unpin
+ Send
+ 'static;
/// Fetch remote header.
fn remote_header(&self, request: RemoteHeaderRequest<Block::Header>) -> Self::RemoteHeaderResult;
/// Fetch remote storage value.
fn remote_read(
fn remote_header(
&self,
request: RemoteReadRequest<Block::Header>
) -> Self::RemoteReadResult;
request: RemoteHeaderRequest<Block::Header>,
) -> Self::RemoteHeaderResult;
/// Fetch remote storage value.
fn remote_read(&self, request: RemoteReadRequest<Block::Header>) -> Self::RemoteReadResult;
/// Fetch remote storage child value.
fn remote_read_child(
&self,
request: RemoteReadChildRequest<Block::Header>
request: RemoteReadChildRequest<Block::Header>,
) -> Self::RemoteReadResult;
/// Fetch remote call result.
fn remote_call(&self, request: RemoteCallRequest<Block::Header>) -> Self::RemoteCallResult;
/// Fetch remote changes ((block number, extrinsic index)) where given key has been changed
/// at a given blocks range.
fn remote_changes(&self, request: RemoteChangesRequest<Block::Header>) -> Self::RemoteChangesResult;
fn remote_changes(
&self,
request: RemoteChangesRequest<Block::Header>,
) -> Self::RemoteChangesResult;
/// Fetch remote block body
fn remote_body(&self, request: RemoteBodyRequest<Block::Header>) -> Self::RemoteBodyResult;
}
@@ -222,20 +225,22 @@ pub trait FetchChecker<Block: BlockT>: Send + Sync {
fn check_changes_proof(
&self,
request: &RemoteChangesRequest<Block::Header>,
proof: ChangesProof<Block::Header>
proof: ChangesProof<Block::Header>,
) -> ClientResult<Vec<(NumberFor<Block>, u32)>>;
/// Check remote body proof.
fn check_body_proof(
&self,
request: &RemoteBodyRequest<Block::Header>,
body: Vec<Block::Extrinsic>
body: Vec<Block::Extrinsic>,
) -> ClientResult<Vec<Block::Extrinsic>>;
}
/// Light client blockchain storage.
pub trait Storage<Block: BlockT>: AuxStore + HeaderBackend<Block>
+ HeaderMetadata<Block, Error=ClientError> + ProvideChtRoots<Block>
pub trait Storage<Block: BlockT>:
AuxStore
+ HeaderBackend<Block>
+ HeaderMetadata<Block, Error = ClientError>
+ ProvideChtRoots<Block>
{
/// Store new header. Should refuse to revert any finalized blocks.
///
@@ -280,10 +285,10 @@ pub enum LocalOrRemote<Data, Request> {
/// locally, or fetches required data from remote node.
pub trait RemoteBlockchain<Block: BlockT>: Send + Sync {
/// Get block header.
fn header(&self, id: BlockId<Block>) -> ClientResult<LocalOrRemote<
Block::Header,
RemoteHeaderRequest<Block::Header>,
>>;
fn header(
&self,
id: BlockId<Block>,
) -> ClientResult<LocalOrRemote<Block::Header, RemoteHeaderRequest<Block::Header>>>;
}
/// Returns future that resolves header either locally, or remotely.
@@ -295,11 +300,8 @@ pub fn future_header<Block: BlockT, F: Fetcher<Block>>(
use futures::future::{ready, Either, FutureExt};
match blockchain.header(id) {
Ok(LocalOrRemote::Remote(request)) => Either::Left(
fetcher
.remote_header(request)
.then(|header| ready(header.map(Some)))
),
Ok(LocalOrRemote::Remote(request)) =>
Either::Left(fetcher.remote_header(request).then(|header| ready(header.map(Some)))),
Ok(LocalOrRemote::Unknown) => Either::Right(ready(Ok(None))),
Ok(LocalOrRemote::Local(local_header)) => Either::Right(ready(Ok(Some(local_header)))),
Err(err) => Either::Right(ready(Err(err))),
@@ -308,11 +310,11 @@ pub fn future_header<Block: BlockT, F: Fetcher<Block>>(
#[cfg(test)]
pub mod tests {
use super::*;
use futures::future::Ready;
use parking_lot::Mutex;
use sp_blockchain::Error as ClientError;
use sp_test_primitives::{Block, Header, Extrinsic};
use super::*;
use sp_test_primitives::{Block, Extrinsic, Header};
#[derive(Debug, thiserror::Error)]
#[error("Not implemented on test node")]
@@ -322,12 +324,11 @@ pub mod tests {
fn into(self) -> ClientError {
ClientError::Application(Box::new(self))
}
}
}
pub type OkCallFetcher = Mutex<Vec<u8>>;
fn not_implemented_in_tests<T>() -> Ready<Result<T, ClientError>>
{
fn not_implemented_in_tests<T>() -> Ready<Result<T, ClientError>> {
futures::future::ready(Err(MockError.into()))
}
@@ -346,7 +347,10 @@ pub mod tests {
not_implemented_in_tests()
}
fn remote_read_child(&self, _request: RemoteReadChildRequest<Header>) -> Self::RemoteReadResult {
fn remote_read_child(
&self,
_request: RemoteReadChildRequest<Header>,
) -> Self::RemoteReadResult {
not_implemented_in_tests()
}
@@ -354,7 +358,10 @@ pub mod tests {
futures::future::ready(Ok((*self.lock()).clone()))
}
fn remote_changes(&self, _request: RemoteChangesRequest<Header>) -> Self::RemoteChangesResult {
fn remote_changes(
&self,
_request: RemoteChangesRequest<Header>,
) -> Self::RemoteChangesResult {
not_implemented_in_tests()
}
+154 -130
View File
@@ -19,15 +19,15 @@
//! Storage notifications
use std::{
collections::{HashSet, HashMap},
collections::{HashMap, HashSet},
sync::Arc,
};
use fnv::{FnvHashSet, FnvHashMap};
use sp_core::storage::{StorageKey, StorageData};
use fnv::{FnvHashMap, FnvHashSet};
use prometheus_endpoint::{register, CounterVec, Opts, Registry, U64};
use sp_core::storage::{StorageData, StorageKey};
use sp_runtime::traits::Block as BlockT;
use sp_utils::mpsc::{TracingUnboundedSender, TracingUnboundedReceiver, tracing_unbounded};
use prometheus_endpoint::{Registry, CounterVec, Opts, U64, register};
use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender};
/// Storage change set
#[derive(Debug)]
@@ -40,29 +40,34 @@ pub struct StorageChangeSet {
impl StorageChangeSet {
/// Convert the change set into iterator over storage items.
pub fn iter<'a>(&'a self)
-> impl Iterator<Item=(Option<&'a StorageKey>, &'a StorageKey, Option<&'a StorageData>)> + 'a {
let top = self.changes
pub fn iter<'a>(
&'a self,
) -> impl Iterator<Item = (Option<&'a StorageKey>, &'a StorageKey, Option<&'a StorageData>)> + 'a
{
let top = self
.changes
.iter()
.filter(move |&(key, _)| match self.filter {
Some(ref filter) => filter.contains(key),
None => true,
})
.map(move |(k,v)| (None, k, v.as_ref()));
let children = self.child_changes
.map(move |(k, v)| (None, k, v.as_ref()));
let children = self
.child_changes
.iter()
.filter_map(move |(sk, changes)|
self.child_filters.as_ref().and_then(|cf|
cf.get(sk).map(|filter| changes
.filter_map(move |(sk, changes)| {
self.child_filters.as_ref().and_then(|cf| {
cf.get(sk).map(|filter| {
changes
.iter()
.filter(move |&(key, _)| match filter {
Some(ref filter) => filter.contains(key),
None => true,
})
.map(move |(k,v)| (Some(sk), k, v.as_ref()))
)
)
)
.map(move |(k, v)| (Some(sk), k, v.as_ref()))
})
})
})
.flatten();
top.chain(children)
}
@@ -82,15 +87,18 @@ pub struct StorageNotifications<Block: BlockT> {
next_id: SubscriberId,
wildcard_listeners: FnvHashSet<SubscriberId>,
listeners: HashMap<StorageKey, FnvHashSet<SubscriberId>>,
child_listeners: HashMap<StorageKey, (
HashMap<StorageKey, FnvHashSet<SubscriberId>>,
FnvHashSet<SubscriberId>
)>,
sinks: FnvHashMap<SubscriberId, (
TracingUnboundedSender<(Block::Hash, StorageChangeSet)>,
Option<HashSet<StorageKey>>,
Option<HashMap<StorageKey, Option<HashSet<StorageKey>>>>,
)>,
child_listeners: HashMap<
StorageKey,
(HashMap<StorageKey, FnvHashSet<SubscriberId>>, FnvHashSet<SubscriberId>),
>,
sinks: FnvHashMap<
SubscriberId,
(
TracingUnboundedSender<(Block::Hash, StorageChangeSet)>,
Option<HashSet<StorageKey>>,
Option<HashMap<StorageKey, Option<HashSet<StorageKey>>>>,
),
>,
}
impl<Block: BlockT> Default for StorageNotifications<Block> {
@@ -110,16 +118,17 @@ impl<Block: BlockT> StorageNotifications<Block> {
/// Initialize a new StorageNotifications
/// optionally pass a prometheus registry to send subscriber metrics to
pub fn new(prometheus_registry: Option<Registry>) -> Self {
let metrics = prometheus_registry.and_then(|r|
let metrics = prometheus_registry.and_then(|r| {
CounterVec::new(
Opts::new(
"storage_notification_subscribers",
"Number of subscribers in storage notification sytem"
"Number of subscribers in storage notification sytem",
),
&["action"], //added | removed
).and_then(|g| register(g, &r))
&["action"], // added | removed
)
.and_then(|g| register(g, &r))
.ok()
);
});
StorageNotifications {
metrics,
@@ -137,17 +146,16 @@ impl<Block: BlockT> StorageNotifications<Block> {
pub fn trigger(
&mut self,
hash: &Block::Hash,
changeset: impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
changeset: impl Iterator<Item = (Vec<u8>, Option<Vec<u8>>)>,
child_changeset: impl Iterator<
Item=(Vec<u8>, impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)>)
Item = (Vec<u8>, impl Iterator<Item = (Vec<u8>, Option<Vec<u8>>)>),
>,
) {
let has_wildcard = !self.wildcard_listeners.is_empty();
// early exit if no listeners
if !has_wildcard && self.listeners.is_empty() && self.child_listeners.is_empty() {
return;
return
}
let mut subscribers = self.wildcard_listeners.clone();
@@ -193,24 +201,29 @@ impl<Block: BlockT> StorageNotifications<Block> {
// Don't send empty notifications
if changes.is_empty() && child_changes.is_empty() {
return;
return
}
let changes = Arc::new(changes);
let child_changes = Arc::new(child_changes);
// Trigger the events
let to_remove = self.sinks
let to_remove = self
.sinks
.iter()
.filter_map(|(subscriber, &(ref sink, ref filter, ref child_filters))| {
let should_remove = {
if subscribers.contains(subscriber) {
sink.unbounded_send((hash.clone(), StorageChangeSet {
changes: changes.clone(),
child_changes: child_changes.clone(),
filter: filter.clone(),
child_filters: child_filters.clone(),
})).is_err()
sink.unbounded_send((
hash.clone(),
StorageChangeSet {
changes: changes.clone(),
child_changes: child_changes.clone(),
filter: filter.clone(),
child_filters: child_filters.clone(),
},
))
.is_err()
} else {
sink.is_closed()
}
@@ -221,7 +234,8 @@ impl<Block: BlockT> StorageNotifications<Block> {
} else {
None
}
}).collect::<Vec<_>>();
})
.collect::<Vec<_>>();
for sub_id in to_remove {
self.remove_subscriber(sub_id);
@@ -233,13 +247,12 @@ impl<Block: BlockT> StorageNotifications<Block> {
filters: &Option<HashSet<StorageKey>>,
listeners: &mut HashMap<StorageKey, FnvHashSet<SubscriberId>>,
wildcards: &mut FnvHashSet<SubscriberId>,
){
) {
match filters {
None => {
wildcards.remove(subscriber);
},
Some(filters) => {
Some(filters) =>
for key in filters.iter() {
let remove_key = match listeners.get_mut(key) {
Some(ref mut set) => {
@@ -252,8 +265,7 @@ impl<Block: BlockT> StorageNotifications<Block> {
if remove_key {
listeners.remove(key);
}
}
}
},
}
}
@@ -267,7 +279,6 @@ impl<Block: BlockT> StorageNotifications<Block> {
);
if let Some(child_filters) = child_filters.as_ref() {
for (c_key, filters) in child_filters {
if let Some((listeners, wildcards)) = self.child_listeners.get_mut(&c_key) {
Self::remove_subscriber_from(
&subscriber,
@@ -293,20 +304,24 @@ impl<Block: BlockT> StorageNotifications<Block> {
filter_keys: &Option<impl AsRef<[StorageKey]>>,
listeners: &mut HashMap<StorageKey, FnvHashSet<SubscriberId>>,
wildcards: &mut FnvHashSet<SubscriberId>,
) -> Option<HashSet<StorageKey>>
{
) -> Option<HashSet<StorageKey>> {
match filter_keys {
None => {
wildcards.insert(current_id);
None
},
Some(keys) => Some(keys.as_ref().iter().map(|key| {
listeners
.entry(key.clone())
.or_insert_with(Default::default)
.insert(current_id);
key.clone()
}).collect())
Some(keys) => Some(
keys.as_ref()
.iter()
.map(|key| {
listeners
.entry(key.clone())
.or_insert_with(Default::default)
.insert(current_id);
key.clone()
})
.collect(),
),
}
}
@@ -327,21 +342,20 @@ impl<Block: BlockT> StorageNotifications<Block> {
&mut self.wildcard_listeners,
);
let child_keys = filter_child_keys.map(|filter_child_keys| {
filter_child_keys.iter().map(|(c_key, o_keys)| {
let (c_listeners, c_wildcards) = self.child_listeners
.entry(c_key.clone())
.or_insert_with(Default::default);
filter_child_keys
.iter()
.map(|(c_key, o_keys)| {
let (c_listeners, c_wildcards) =
self.child_listeners.entry(c_key.clone()).or_insert_with(Default::default);
(c_key.clone(), Self::listen_from(
current_id,
o_keys,
&mut *c_listeners,
&mut *c_wildcards,
))
}).collect()
(
c_key.clone(),
Self::listen_from(current_id, o_keys, &mut *c_listeners, &mut *c_wildcards),
)
})
.collect()
});
// insert sink
let (tx, rx) = tracing_unbounded("mpsc_storage_notification_items");
self.sinks.insert(current_id, (tx, keys, child_keys));
@@ -356,8 +370,8 @@ impl<Block: BlockT> StorageNotifications<Block> {
#[cfg(test)]
mod tests {
use sp_runtime::testing::{H256 as Hash, Block as RawBlock, ExtrinsicWrapper};
use super::*;
use sp_runtime::testing::{Block as RawBlock, ExtrinsicWrapper, H256 as Hash};
use std::iter::{empty, Empty};
type TestChangeSet = (
@@ -369,10 +383,12 @@ mod tests {
impl From<TestChangeSet> for StorageChangeSet {
fn from(changes: TestChangeSet) -> Self {
// warning hardcoded child trie wildcard to test upon
let child_filters = Some([
(StorageKey(vec![4]), None),
(StorageKey(vec![5]), None),
].iter().cloned().collect());
let child_filters = Some(
[(StorageKey(vec![4]), None), (StorageKey(vec![5]), None)]
.iter()
.cloned()
.collect(),
);
StorageChangeSet {
changes: Arc::new(changes.0),
child_changes: Arc::new(changes.1),
@@ -396,34 +412,40 @@ mod tests {
// given
let mut notifications = StorageNotifications::<Block>::default();
let child_filter = [(StorageKey(vec![4]), None)];
let mut recv = futures::executor::block_on_stream(
notifications.listen(None, Some(&child_filter[..]))
);
let mut recv =
futures::executor::block_on_stream(notifications.listen(None, Some(&child_filter[..])));
// when
let changeset = vec![
(vec![2], Some(vec![3])),
(vec![3], None),
];
let c_changeset_1 = vec![
(vec![5], Some(vec![4])),
(vec![6], None),
];
let changeset = vec![(vec![2], Some(vec![3])), (vec![3], None)];
let c_changeset_1 = vec![(vec![5], Some(vec![4])), (vec![6], None)];
let c_changeset = vec![(vec![4], c_changeset_1)];
notifications.trigger(
&Hash::from_low_u64_be(1),
changeset.into_iter(),
c_changeset.into_iter().map(|(a,b)| (a, b.into_iter())),
c_changeset.into_iter().map(|(a, b)| (a, b.into_iter())),
);
// then
assert_eq!(recv.next().unwrap(), (Hash::from_low_u64_be(1), (vec![
(StorageKey(vec![2]), Some(StorageData(vec![3]))),
(StorageKey(vec![3]), None),
], vec![(StorageKey(vec![4]), vec![
(StorageKey(vec![5]), Some(StorageData(vec![4]))),
(StorageKey(vec![6]), None),
])]).into()));
assert_eq!(
recv.next().unwrap(),
(
Hash::from_low_u64_be(1),
(
vec![
(StorageKey(vec![2]), Some(StorageData(vec![3]))),
(StorageKey(vec![3]), None),
],
vec![(
StorageKey(vec![4]),
vec![
(StorageKey(vec![5]), Some(StorageData(vec![4]))),
(StorageKey(vec![6]), None),
]
)]
)
.into()
)
);
}
#[test]
@@ -432,44 +454,52 @@ mod tests {
let mut notifications = StorageNotifications::<Block>::default();
let child_filter = [(StorageKey(vec![4]), Some(vec![StorageKey(vec![5])]))];
let mut recv1 = futures::executor::block_on_stream(
notifications.listen(Some(&[StorageKey(vec![1])]), None)
notifications.listen(Some(&[StorageKey(vec![1])]), None),
);
let mut recv2 = futures::executor::block_on_stream(
notifications.listen(Some(&[StorageKey(vec![2])]), None)
notifications.listen(Some(&[StorageKey(vec![2])]), None),
);
let mut recv3 = futures::executor::block_on_stream(
notifications.listen(Some(&[]), Some(&child_filter))
notifications.listen(Some(&[]), Some(&child_filter)),
);
// when
let changeset = vec![
(vec![2], Some(vec![3])),
(vec![1], None),
];
let c_changeset_1 = vec![
(vec![5], Some(vec![4])),
(vec![6], None),
];
let changeset = vec![(vec![2], Some(vec![3])), (vec![1], None)];
let c_changeset_1 = vec![(vec![5], Some(vec![4])), (vec![6], None)];
let c_changeset = vec![(vec![4], c_changeset_1)];
notifications.trigger(
&Hash::from_low_u64_be(1),
changeset.into_iter(),
c_changeset.into_iter().map(|(a,b)| (a, b.into_iter())),
c_changeset.into_iter().map(|(a, b)| (a, b.into_iter())),
);
// then
assert_eq!(recv1.next().unwrap(), (Hash::from_low_u64_be(1), (vec![
(StorageKey(vec![1]), None),
], vec![]).into()));
assert_eq!(recv2.next().unwrap(), (Hash::from_low_u64_be(1), (vec![
(StorageKey(vec![2]), Some(StorageData(vec![3]))),
], vec![]).into()));
assert_eq!(recv3.next().unwrap(), (Hash::from_low_u64_be(1), (vec![],
vec![
(StorageKey(vec![4]), vec![(StorageKey(vec![5]), Some(StorageData(vec![4])))]),
]).into()));
assert_eq!(
recv1.next().unwrap(),
(Hash::from_low_u64_be(1), (vec![(StorageKey(vec![1]), None),], vec![]).into())
);
assert_eq!(
recv2.next().unwrap(),
(
Hash::from_low_u64_be(1),
(vec![(StorageKey(vec![2]), Some(StorageData(vec![3]))),], vec![]).into()
)
);
assert_eq!(
recv3.next().unwrap(),
(
Hash::from_low_u64_be(1),
(
vec![],
vec![(
StorageKey(vec![4]),
vec![(StorageKey(vec![5]), Some(StorageData(vec![4])))]
),]
)
.into()
)
);
}
#[test]
@@ -479,27 +509,21 @@ mod tests {
{
let child_filter = [(StorageKey(vec![4]), Some(vec![StorageKey(vec![5])]))];
let _recv1 = futures::executor::block_on_stream(
notifications.listen(Some(&[StorageKey(vec![1])]), None)
notifications.listen(Some(&[StorageKey(vec![1])]), None),
);
let _recv2 = futures::executor::block_on_stream(
notifications.listen(Some(&[StorageKey(vec![2])]), None)
);
let _recv3 = futures::executor::block_on_stream(
notifications.listen(None, None)
);
let _recv4 = futures::executor::block_on_stream(
notifications.listen(None, Some(&child_filter))
notifications.listen(Some(&[StorageKey(vec![2])]), None),
);
let _recv3 = futures::executor::block_on_stream(notifications.listen(None, None));
let _recv4 =
futures::executor::block_on_stream(notifications.listen(None, Some(&child_filter)));
assert_eq!(notifications.listeners.len(), 2);
assert_eq!(notifications.wildcard_listeners.len(), 2);
assert_eq!(notifications.child_listeners.len(), 1);
}
// when
let changeset = vec![
(vec![2], Some(vec![3])),
(vec![1], None),
];
let changeset = vec![(vec![2], Some(vec![3])), (vec![1], None)];
let c_changeset = empty::<(_, Empty<_>)>();
notifications.trigger(&Hash::from_low_u64_be(1), changeset.into_iter(), c_changeset);
+9 -9
View File
@@ -17,12 +17,9 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//! Proof utilities
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT},
};
use crate::{StorageProof, ChangesProof};
use sp_storage::{ChildInfo, StorageKey, PrefixedStorageKey};
use crate::{ChangesProof, StorageProof};
use sp_runtime::{generic::BlockId, traits::Block as BlockT};
use sp_storage::{ChildInfo, PrefixedStorageKey, StorageKey};
/// Interface for providing block proving utilities.
pub trait ProofProvider<Block: BlockT> {
@@ -30,7 +27,7 @@ pub trait ProofProvider<Block: BlockT> {
fn read_proof(
&self,
id: &BlockId<Block>,
keys: &mut dyn Iterator<Item=&[u8]>,
keys: &mut dyn Iterator<Item = &[u8]>,
) -> sp_blockchain::Result<StorageProof>;
/// Reads child storage value at a given block + storage_key + key, returning
@@ -39,7 +36,7 @@ pub trait ProofProvider<Block: BlockT> {
&self,
id: &BlockId<Block>,
child_info: &ChildInfo,
keys: &mut dyn Iterator<Item=&[u8]>,
keys: &mut dyn Iterator<Item = &[u8]>,
) -> sp_blockchain::Result<StorageProof>;
/// Execute a call to a contract on top of state in a block of given hash
@@ -53,7 +50,10 @@ pub trait ProofProvider<Block: BlockT> {
call_data: &[u8],
) -> sp_blockchain::Result<(Vec<u8>, StorageProof)>;
/// Reads given header and generates CHT-based header proof.
fn header_proof(&self, id: &BlockId<Block>) -> sp_blockchain::Result<(Block::Header, StorageProof)>;
fn header_proof(
&self,
id: &BlockId<Block>,
) -> sp_blockchain::Result<(Block::Header, StorageProof)>;
/// Get proof for computation of (block, extrinsic) pairs where key has been changed at given blocks range.
/// `min` is the hash of the first block, which changes trie root is known to the requester - when we're using
@@ -16,13 +16,13 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use futures::stream::Stream;
use futures::future::FutureExt;
use futures::ready;
use futures::{future::FutureExt, ready, stream::Stream};
use futures_timer::Delay;
use std::pin::Pin;
use std::task::{Context, Poll};
use std::time::Duration;
use std::{
pin::Pin,
task::{Context, Poll},
time::Duration,
};
/// Exponentially increasing interval
///
@@ -37,11 +37,7 @@ impl ExpIncInterval {
/// Create a new [`ExpIncInterval`].
pub fn new(start: Duration, max: Duration) -> Self {
let delay = Delay::new(start);
Self {
max,
next: start * 2,
delay,
}
Self { max, next: start * 2, delay }
}
/// Fast forward the exponentially increasing interval to the configured maximum.
+12 -14
View File
@@ -26,18 +26,23 @@
//!
//! See [`Worker`] and [`Service`] for more documentation.
pub use crate::{service::Service, worker::{NetworkProvider, Worker, Role}};
pub use crate::{
service::Service,
worker::{NetworkProvider, Role, Worker},
};
use std::{sync::Arc, time::Duration};
use futures::channel::{mpsc, oneshot};
use futures::Stream;
use futures::{
channel::{mpsc, oneshot},
Stream,
};
use sc_client_api::blockchain::HeaderBackend;
use sc_network::{DhtEvent, Multiaddr, PeerId};
use sp_api::ProvideRuntimeApi;
use sp_authority_discovery::{AuthorityDiscoveryApi, AuthorityId};
use sp_runtime::traits::Block as BlockT;
use sp_api::ProvideRuntimeApi;
mod error;
mod interval;
@@ -141,15 +146,8 @@ where
{
let (to_worker, from_service) = mpsc::channel(0);
let worker = Worker::new(
from_service,
client,
network,
dht_event_rx,
role,
prometheus_registry,
config,
);
let worker =
Worker::new(from_service, client, network, dht_event_rx, role, prometheus_registry, config);
let service = Service::new(to_worker);
(worker, service)
@@ -160,5 +158,5 @@ pub(crate) enum ServicetoWorkerMsg {
/// See [`Service::get_addresses_by_authority_id`].
GetAddressesByAuthorityId(AuthorityId, oneshot::Sender<Option<Vec<Multiaddr>>>),
/// See [`Service::get_authority_id_by_peer_id`].
GetAuthorityIdByPeerId(PeerId, oneshot::Sender<Option<AuthorityId>>)
GetAuthorityIdByPeerId(PeerId, oneshot::Sender<Option<AuthorityId>>),
}
@@ -20,8 +20,10 @@ use std::fmt::Debug;
use crate::ServicetoWorkerMsg;
use futures::channel::{mpsc, oneshot};
use futures::SinkExt;
use futures::{
channel::{mpsc, oneshot},
SinkExt,
};
use sc_network::{Multiaddr, PeerId};
use sp_authority_discovery::AuthorityId;
@@ -42,9 +44,7 @@ impl Debug for Service {
/// [`crate::Worker`]'s local address cache for a given [`AuthorityId`].
impl Service {
pub(crate) fn new(to_worker: mpsc::Sender<ServicetoWorkerMsg>) -> Self {
Self {
to_worker,
}
Self { to_worker }
}
/// Get the addresses for the given [`AuthorityId`] from the local address
@@ -59,7 +59,10 @@ impl Service {
/// enforced today, given that there are still authorities out there
/// publishing the addresses of their sentry nodes on the DHT. In the future
/// this guarantee can be provided.
pub async fn get_addresses_by_authority_id(&mut self, authority: AuthorityId) -> Option<Vec<Multiaddr>> {
pub async fn get_addresses_by_authority_id(
&mut self,
authority: AuthorityId,
) -> Option<Vec<Multiaddr>> {
let (tx, rx) = oneshot::channel();
self.to_worker
@@ -16,15 +16,24 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{new_worker_and_service, worker::{tests::{TestApi, TestNetwork}, Role}};
use crate::{
new_worker_and_service,
worker::{
tests::{TestApi, TestNetwork},
Role,
},
};
use std::sync::Arc;
use futures::{channel::mpsc::channel, executor::LocalPool, task::LocalSpawn};
use libp2p::core::{multiaddr::{Multiaddr, Protocol}, PeerId};
use libp2p::core::{
multiaddr::{Multiaddr, Protocol},
PeerId,
};
use std::sync::Arc;
use sp_authority_discovery::AuthorityId;
use sp_core::crypto::key_types;
use sp_keystore::{CryptoStore, testing::KeyStore};
use sp_keystore::{testing::KeyStore, CryptoStore};
#[test]
fn get_addresses_and_authority_id() {
@@ -44,13 +53,12 @@ fn get_addresses_and_authority_id() {
});
let remote_peer_id = PeerId::random();
let remote_addr = "/ip6/2001:db8:0:0:0:0:0:2/tcp/30333".parse::<Multiaddr>()
let remote_addr = "/ip6/2001:db8:0:0:0:0:0:2/tcp/30333"
.parse::<Multiaddr>()
.unwrap()
.with(Protocol::P2p(remote_peer_id.clone().into()));
let test_api = Arc::new(TestApi {
authorities: vec![],
});
let test_api = Arc::new(TestApi { authorities: vec![] });
let (mut worker, mut service) = new_worker_and_service(
test_api,
+125 -145
View File
@@ -16,43 +16,49 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{error::{Error, Result}, interval::ExpIncInterval, ServicetoWorkerMsg};
use crate::{
error::{Error, Result},
interval::ExpIncInterval,
ServicetoWorkerMsg,
};
use std::collections::{HashMap, HashSet};
use std::convert::TryInto;
use std::marker::PhantomData;
use std::sync::Arc;
use std::time::Duration;
use std::{
collections::{HashMap, HashSet},
convert::TryInto,
marker::PhantomData,
sync::Arc,
time::Duration,
};
use futures::channel::mpsc;
use futures::{future, FutureExt, Stream, StreamExt, stream::Fuse};
use futures::{channel::mpsc, future, stream::Fuse, FutureExt, Stream, StreamExt};
use addr_cache::AddrCache;
use async_trait::async_trait;
use codec::Decode;
use ip_network::IpNetwork;
use libp2p::{core::multiaddr, multihash::{Multihash, Hasher}};
use libp2p::{
core::multiaddr,
multihash::{Hasher, Multihash},
};
use log::{debug, error, log_enabled};
use prometheus_endpoint::{Counter, CounterVec, Gauge, Opts, U64, register};
use prometheus_endpoint::{register, Counter, CounterVec, Gauge, Opts, U64};
use prost::Message;
use rand::{seq::SliceRandom, thread_rng};
use sc_client_api::blockchain::HeaderBackend;
use sc_network::{
DhtEvent,
ExHashT,
Multiaddr,
NetworkStateInfo,
PeerId,
use sc_network::{DhtEvent, ExHashT, Multiaddr, NetworkStateInfo, PeerId};
use sp_api::ProvideRuntimeApi;
use sp_authority_discovery::{
AuthorityDiscoveryApi, AuthorityId, AuthorityPair, AuthoritySignature,
};
use sp_authority_discovery::{AuthorityDiscoveryApi, AuthorityId, AuthoritySignature, AuthorityPair};
use sp_core::crypto::{key_types, CryptoTypePublicPair, Pair};
use sp_keystore::CryptoStore;
use sp_runtime::{traits::Block as BlockT, generic::BlockId};
use sp_api::ProvideRuntimeApi;
use sp_runtime::{generic::BlockId, traits::Block as BlockT};
mod addr_cache;
/// Dht payload schemas generated from Protobuf definitions via Prost crate in build.rs.
mod schema { include!(concat!(env!("OUT_DIR"), "/authority_discovery.rs")); }
mod schema {
include!(concat!(env!("OUT_DIR"), "/authority_discovery.rs"));
}
#[cfg(test)]
pub mod tests;
@@ -72,7 +78,6 @@ pub enum Role {
Discover,
}
/// An authority discovery [`Worker`] can publish the local node's addresses as well as discover
/// those of other nodes via a Kademlia DHT.
///
@@ -141,8 +146,7 @@ where
Block: BlockT + Unpin + 'static,
Network: NetworkProvider,
Client: ProvideRuntimeApi<Block> + Send + Sync + 'static + HeaderBackend<Block>,
<Client as ProvideRuntimeApi<Block>>::Api:
AuthorityDiscoveryApi<Block>,
<Client as ProvideRuntimeApi<Block>>::Api: AuthorityDiscoveryApi<Block>,
DhtEventStream: Stream<Item = DhtEvent> + Unpin,
{
/// Construct a [`Worker`].
@@ -161,33 +165,24 @@ where
// thus timely retries are not needed. For this reasoning use an exponentially increasing
// interval for `publish_interval`, `query_interval` and `priority_group_set_interval`
// instead of a constant interval.
let publish_interval = ExpIncInterval::new(
Duration::from_secs(2),
config.max_publish_interval,
);
let query_interval = ExpIncInterval::new(
Duration::from_secs(2),
config.max_query_interval,
);
let publish_interval =
ExpIncInterval::new(Duration::from_secs(2), config.max_publish_interval);
let query_interval = ExpIncInterval::new(Duration::from_secs(2), config.max_query_interval);
// An `ExpIncInterval` is overkill here because the interval is constant, but consistency
// is more simple.
let publish_if_changed_interval = ExpIncInterval::new(
config.keystore_refresh_interval,
config.keystore_refresh_interval
);
let publish_if_changed_interval =
ExpIncInterval::new(config.keystore_refresh_interval, config.keystore_refresh_interval);
let addr_cache = AddrCache::new();
let metrics = match prometheus_registry {
Some(registry) => {
match Metrics::register(&registry) {
Ok(metrics) => Some(metrics),
Err(e) => {
error!(target: LOG_TARGET, "Failed to register metrics: {:?}", e);
None
},
}
Some(registry) => match Metrics::register(&registry) {
Ok(metrics) => Some(metrics),
Err(e) => {
error!(target: LOG_TARGET, "Failed to register metrics: {:?}", e);
None
},
},
None => None,
};
@@ -262,23 +257,23 @@ where
let _ = sender.send(
self.addr_cache.get_addresses_by_authority_id(&authority).map(Clone::clone),
);
}
},
ServicetoWorkerMsg::GetAuthorityIdByPeerId(peer_id, sender) => {
let _ = sender.send(
self.addr_cache.get_authority_id_by_peer_id(&peer_id).map(Clone::clone),
);
}
let _ = sender
.send(self.addr_cache.get_authority_id_by_peer_id(&peer_id).map(Clone::clone));
},
}
}
fn addresses_to_publish(&self) -> impl Iterator<Item = Multiaddr> {
let peer_id: Multihash = self.network.local_peer_id().into();
let publish_non_global_ips = self.publish_non_global_ips;
self.network.external_addresses()
self.network
.external_addresses()
.into_iter()
.filter(move |a| {
if publish_non_global_ips {
return true;
return true
}
a.iter().all(|p| match p {
@@ -321,9 +316,9 @@ where
if let Some(metrics) = &self.metrics {
metrics.publish.inc();
metrics.amount_addresses_last_published.set(
addresses.len().try_into().unwrap_or(std::u64::MAX),
);
metrics
.amount_addresses_last_published
.set(addresses.len().try_into().unwrap_or(std::u64::MAX));
}
let mut serialized_addresses = vec![];
@@ -332,30 +327,26 @@ where
.map_err(Error::EncodingProto)?;
let keys_vec = keys.iter().cloned().collect::<Vec<_>>();
let signatures = key_store.sign_with_all(
key_types::AUTHORITY_DISCOVERY,
keys_vec.clone(),
serialized_addresses.as_slice(),
).await.map_err(|_| Error::Signing)?;
let signatures = key_store
.sign_with_all(
key_types::AUTHORITY_DISCOVERY,
keys_vec.clone(),
serialized_addresses.as_slice(),
)
.await
.map_err(|_| Error::Signing)?;
for (sign_result, key) in signatures.into_iter().zip(keys_vec.iter()) {
let mut signed_addresses = vec![];
// Verify that all signatures exist for all provided keys.
let signature = sign_result.ok()
.flatten()
.ok_or_else(|| Error::MissingSignature(key.clone()))?;
schema::SignedAuthorityAddresses {
addresses: serialized_addresses.clone(),
signature,
}
.encode(&mut signed_addresses)
let signature =
sign_result.ok().flatten().ok_or_else(|| Error::MissingSignature(key.clone()))?;
schema::SignedAuthorityAddresses { addresses: serialized_addresses.clone(), signature }
.encode(&mut signed_addresses)
.map_err(Error::EncodingProto)?;
self.network.put_value(
hash_authority_id(key.1.as_ref()),
signed_addresses,
);
self.network.put_value(hash_authority_id(key.1.as_ref()), signed_addresses);
}
self.latest_published_keys = keys;
@@ -367,11 +358,11 @@ where
let id = BlockId::hash(self.client.info().best_hash);
let local_keys = match &self.role {
Role::PublishAndDiscover(key_store) => {
key_store.sr25519_public_keys(
key_types::AUTHORITY_DISCOVERY
).await.into_iter().collect::<HashSet<_>>()
},
Role::PublishAndDiscover(key_store) => key_store
.sr25519_public_keys(key_types::AUTHORITY_DISCOVERY)
.await
.into_iter()
.collect::<HashSet<_>>(),
Role::Discover => HashSet::new(),
};
@@ -393,9 +384,9 @@ where
self.in_flight_lookups.clear();
if let Some(metrics) = &self.metrics {
metrics.requests_pending.set(
self.pending_lookups.len().try_into().unwrap_or(std::u64::MAX),
);
metrics
.requests_pending
.set(self.pending_lookups.len().try_into().unwrap_or(std::u64::MAX));
}
Ok(())
@@ -408,15 +399,14 @@ where
None => return,
};
let hash = hash_authority_id(authority_id.as_ref());
self.network
.get_value(&hash);
self.network.get_value(&hash);
self.in_flight_lookups.insert(hash, authority_id);
if let Some(metrics) = &self.metrics {
metrics.requests.inc();
metrics.requests_pending.set(
self.pending_lookups.len().try_into().unwrap_or(std::u64::MAX),
);
metrics
.requests_pending
.set(self.pending_lookups.len().try_into().unwrap_or(std::u64::MAX));
}
}
}
@@ -431,10 +421,7 @@ where
if log_enabled!(log::Level::Debug) {
let hashes: Vec<_> = v.iter().map(|(hash, _value)| hash.clone()).collect();
debug!(
target: LOG_TARGET,
"Value for hash '{:?}' found on Dht.", hashes,
);
debug!(target: LOG_TARGET, "Value for hash '{:?}' found on Dht.", hashes,);
}
if let Err(e) = self.handle_dht_value_found_event(v) {
@@ -442,22 +429,16 @@ where
metrics.handle_value_found_event_failure.inc();
}
debug!(
target: LOG_TARGET,
"Failed to handle Dht value found event: {:?}", e,
);
debug!(target: LOG_TARGET, "Failed to handle Dht value found event: {:?}", e,);
}
}
},
DhtEvent::ValueNotFound(hash) => {
if let Some(metrics) = &self.metrics {
metrics.dht_event_received.with_label_values(&["value_not_found"]).inc();
}
if self.in_flight_lookups.remove(&hash).is_some() {
debug!(
target: LOG_TARGET,
"Value for hash '{:?}' not found on Dht.", hash
)
debug!(target: LOG_TARGET, "Value for hash '{:?}' not found on Dht.", hash)
} else {
debug!(
target: LOG_TARGET,
@@ -475,21 +456,15 @@ where
metrics.dht_event_received.with_label_values(&["value_put"]).inc();
}
debug!(
target: LOG_TARGET,
"Successfully put hash '{:?}' on Dht.", hash,
)
debug!(target: LOG_TARGET, "Successfully put hash '{:?}' on Dht.", hash,)
},
DhtEvent::ValuePutFailed(hash) => {
if let Some(metrics) = &self.metrics {
metrics.dht_event_received.with_label_values(&["value_put_failed"]).inc();
}
debug!(
target: LOG_TARGET,
"Failed to put hash '{:?}' on Dht.", hash
)
}
debug!(target: LOG_TARGET, "Failed to put hash '{:?}' on Dht.", hash)
},
}
}
@@ -498,34 +473,36 @@ where
values: Vec<(libp2p::kad::record::Key, Vec<u8>)>,
) -> Result<()> {
// Ensure `values` is not empty and all its keys equal.
let remote_key = values.iter().fold(Ok(None), |acc, (key, _)| {
match acc {
let remote_key = values
.iter()
.fold(Ok(None), |acc, (key, _)| match acc {
Ok(None) => Ok(Some(key.clone())),
Ok(Some(ref prev_key)) if prev_key != key => Err(
Error::ReceivingDhtValueFoundEventWithDifferentKeys
),
Ok(Some(ref prev_key)) if prev_key != key =>
Err(Error::ReceivingDhtValueFoundEventWithDifferentKeys),
x @ Ok(_) => x,
Err(e) => Err(e),
}
})?.ok_or(Error::ReceivingDhtValueFoundEventWithNoRecords)?;
})?
.ok_or(Error::ReceivingDhtValueFoundEventWithNoRecords)?;
let authority_id: AuthorityId = self.in_flight_lookups
let authority_id: AuthorityId = self
.in_flight_lookups
.remove(&remote_key)
.ok_or(Error::ReceivingUnexpectedRecord)?;
let local_peer_id = self.network.local_peer_id();
let remote_addresses: Vec<Multiaddr> = values.into_iter()
let remote_addresses: Vec<Multiaddr> = values
.into_iter()
.map(|(_k, v)| {
let schema::SignedAuthorityAddresses { signature, addresses } =
schema::SignedAuthorityAddresses::decode(v.as_slice())
.map_err(Error::DecodingProto)?;
.map_err(Error::DecodingProto)?;
let signature = AuthoritySignature::decode(&mut &signature[..])
.map_err(Error::EncodingDecodingScale)?;
if !AuthorityPair::verify(&signature, &addresses, &authority_id) {
return Err(Error::VerifyingDhtPayload);
return Err(Error::VerifyingDhtPayload)
}
let addresses = schema::AuthorityAddresses::decode(addresses.as_slice())
@@ -542,40 +519,41 @@ where
.into_iter()
.flatten()
// Ignore [`Multiaddr`]s without [`PeerId`] and own addresses.
.filter(|addr| addr.iter().any(|protocol| {
// Parse to PeerId first as Multihashes of old and new PeerId
// representation don't equal.
//
// See https://github.com/libp2p/rust-libp2p/issues/555 for
// details.
if let multiaddr::Protocol::P2p(hash) = protocol {
let peer_id = match PeerId::from_multihash(hash) {
Ok(peer_id) => peer_id,
Err(_) => return false, // Discard address.
};
.filter(|addr| {
addr.iter().any(|protocol| {
// Parse to PeerId first as Multihashes of old and new PeerId
// representation don't equal.
//
// See https://github.com/libp2p/rust-libp2p/issues/555 for
// details.
if let multiaddr::Protocol::P2p(hash) = protocol {
let peer_id = match PeerId::from_multihash(hash) {
Ok(peer_id) => peer_id,
Err(_) => return false, // Discard address.
};
// Discard if equal to local peer id, keep if it differs.
return !(peer_id == local_peer_id);
}
// Discard if equal to local peer id, keep if it differs.
return !(peer_id == local_peer_id)
}
false // `protocol` is not a [`Protocol::P2p`], let's keep looking.
}))
false // `protocol` is not a [`Protocol::P2p`], let's keep looking.
})
})
.take(MAX_ADDRESSES_PER_AUTHORITY)
.collect();
if !remote_addresses.is_empty() {
self.addr_cache.insert(authority_id, remote_addresses);
if let Some(metrics) = &self.metrics {
metrics.known_authorities_count.set(
self.addr_cache.num_ids().try_into().unwrap_or(std::u64::MAX)
);
metrics
.known_authorities_count
.set(self.addr_cache.num_ids().try_into().unwrap_or(std::u64::MAX));
}
}
Ok(())
}
/// Retrieve our public keys within the current and next authority set.
//
// A node might have multiple authority discovery keys within its keystore, e.g. an old one and
// one for the upcoming session. In addition it could be participating in the current and (/ or)
// next authority set with two keys. The function does not return all of the local authority
@@ -591,14 +569,16 @@ where
.collect::<HashSet<_>>();
let id = BlockId::hash(client.info().best_hash);
let authorities = client.runtime_api()
let authorities = client
.runtime_api()
.authorities(&id)
.map_err(|e| Error::CallingRuntime(e.into()))?
.into_iter()
.map(std::convert::Into::into)
.collect::<HashSet<_>>();
let intersection = local_pub_keys.intersection(&authorities)
let intersection = local_pub_keys
.intersection(&authorities)
.cloned()
.map(std::convert::Into::into)
.collect();
@@ -655,7 +635,7 @@ impl Metrics {
publish: register(
Counter::new(
"authority_discovery_times_published_total",
"Number of times authority discovery has published external addresses."
"Number of times authority discovery has published external addresses.",
)?,
registry,
)?,
@@ -663,7 +643,7 @@ impl Metrics {
Gauge::new(
"authority_discovery_amount_external_addresses_last_published",
"Number of external addresses published when authority discovery last \
published addresses."
published addresses.",
)?,
registry,
)?,
@@ -671,14 +651,14 @@ impl Metrics {
Counter::new(
"authority_discovery_authority_addresses_requested_total",
"Number of times authority discovery has requested external addresses of a \
single authority."
single authority.",
)?,
registry,
)?,
requests_pending: register(
Gauge::new(
"authority_discovery_authority_address_requests_pending",
"Number of pending authority address requests."
"Number of pending authority address requests.",
)?,
registry,
)?,
@@ -686,7 +666,7 @@ impl Metrics {
CounterVec::new(
Opts::new(
"authority_discovery_dht_event_received",
"Number of dht events received by authority discovery."
"Number of dht events received by authority discovery.",
),
&["name"],
)?,
@@ -695,14 +675,14 @@ impl Metrics {
handle_value_found_event_failure: register(
Counter::new(
"authority_discovery_handle_value_found_event_failure",
"Number of times handling a dht value found event failed."
"Number of times handling a dht value found event failed.",
)?,
registry,
)?,
known_authorities_count: register(
Gauge::new(
"authority_discovery_known_authorities_count",
"Number of authorities known by authority discovery."
"Number of authorities known by authority discovery.",
)?,
registry,
)?,
@@ -19,8 +19,8 @@
use libp2p::core::multiaddr::{Multiaddr, Protocol};
use std::collections::HashMap;
use sp_authority_discovery::AuthorityId;
use sc_network::PeerId;
use sp_authority_discovery::AuthorityId;
/// Cache for [`AuthorityId`] -> [`Vec<Multiaddr>`] and [`PeerId`] -> [`AuthorityId`] mappings.
pub(super) struct AddrCache {
@@ -45,27 +45,34 @@ impl AddrCache {
addresses.sort_unstable_by(|a, b| a.as_ref().cmp(b.as_ref()));
// Insert into `self.peer_id_to_authority_id`.
let peer_ids = addresses.iter()
let peer_ids = addresses
.iter()
.map(|a| peer_id_from_multiaddr(a))
.filter_map(|peer_id| peer_id);
for peer_id in peer_ids.clone() {
let former_auth = match self.peer_id_to_authority_id.insert(peer_id, authority_id.clone()) {
Some(a) if a != authority_id => a,
_ => continue,
};
let former_auth =
match self.peer_id_to_authority_id.insert(peer_id, authority_id.clone()) {
Some(a) if a != authority_id => a,
_ => continue,
};
// PeerId was associated to a different authority id before.
// Remove corresponding authority from `self.authority_id_to_addresses`.
let former_auth_addrs = match self.authority_id_to_addresses.get_mut(&former_auth) {
Some(a) => a,
None => { debug_assert!(false); continue }
None => {
debug_assert!(false);
continue
},
};
former_auth_addrs.retain(|a| peer_id_from_multiaddr(a).map_or(true, |p| p != peer_id));
}
// Insert into `self.authority_id_to_addresses`.
for former_addr in
self.authority_id_to_addresses.insert(authority_id.clone(), addresses.clone()).unwrap_or_default()
for former_addr in self
.authority_id_to_addresses
.insert(authority_id.clone(), addresses.clone())
.unwrap_or_default()
{
// Must remove from `self.peer_id_to_authority_id` any PeerId formerly associated
// to that authority but that can't be found in its new addresses.
@@ -87,7 +94,10 @@ impl AddrCache {
}
/// Returns the addresses for the given [`AuthorityId`].
pub fn get_addresses_by_authority_id(&self, authority_id: &AuthorityId) -> Option<&Vec<Multiaddr>> {
pub fn get_addresses_by_authority_id(
&self,
authority_id: &AuthorityId,
) -> Option<&Vec<Multiaddr>> {
self.authority_id_to_addresses.get(&authority_id)
}
@@ -100,7 +110,9 @@ impl AddrCache {
/// [`AuthorityId`]s.
pub fn retain_ids(&mut self, authority_ids: &Vec<AuthorityId>) {
// The below logic could be replaced by `BtreeMap::drain_filter` once it stabilized.
let authority_ids_to_remove = self.authority_id_to_addresses.iter()
let authority_ids_to_remove = self
.authority_id_to_addresses
.iter()
.filter(|(id, _addresses)| !authority_ids.contains(id))
.map(|entry| entry.0)
.cloned()
@@ -111,7 +123,8 @@ impl AddrCache {
let addresses = self.authority_id_to_addresses.remove(&authority_id_to_remove);
// Remove other entries from `self.peer_id_to_authority_id`.
let peer_ids = addresses.iter()
let peer_ids = addresses
.iter()
.flatten()
.map(|a| peer_id_from_multiaddr(a))
.filter_map(|peer_id| peer_id);
@@ -125,10 +138,12 @@ impl AddrCache {
}
fn peer_id_from_multiaddr(addr: &Multiaddr) -> Option<PeerId> {
addr.iter().last().and_then(|protocol| if let Protocol::P2p(multihash) = protocol {
PeerId::from_multihash(multihash).ok()
} else {
None
addr.iter().last().and_then(|protocol| {
if let Protocol::P2p(multihash) = protocol {
PeerId::from_multihash(multihash).ok()
} else {
None
}
})
}
@@ -159,9 +174,11 @@ mod tests {
fn arbitrary(g: &mut Gen) -> Self {
let seed = (0..32).map(|_| u8::arbitrary(g)).collect::<Vec<_>>();
let peer_id = PeerId::from_multihash(
Multihash::wrap(multihash::Code::Sha2_256.into(), &seed).unwrap()
).unwrap();
let multiaddr = "/ip6/2001:db8:0:0:0:0:0:2/tcp/30333".parse::<Multiaddr>()
Multihash::wrap(multihash::Code::Sha2_256.into(), &seed).unwrap(),
)
.unwrap();
let multiaddr = "/ip6/2001:db8:0:0:0:0:0:2/tcp/30333"
.parse::<Multiaddr>()
.unwrap()
.with(Protocol::P2p(peer_id.into()));
@@ -176,12 +193,15 @@ mod tests {
fn arbitrary(g: &mut Gen) -> Self {
let seed = (0..32).map(|_| u8::arbitrary(g)).collect::<Vec<_>>();
let peer_id = PeerId::from_multihash(
Multihash::wrap(multihash::Code::Sha2_256.into(), &seed).unwrap()
).unwrap();
let multiaddr1 = "/ip6/2001:db8:0:0:0:0:0:2/tcp/30333".parse::<Multiaddr>()
Multihash::wrap(multihash::Code::Sha2_256.into(), &seed).unwrap(),
)
.unwrap();
let multiaddr1 = "/ip6/2001:db8:0:0:0:0:0:2/tcp/30333"
.parse::<Multiaddr>()
.unwrap()
.with(Protocol::P2p(peer_id.clone().into()));
let multiaddr2 = "/ip6/2002:db8:0:0:0:0:0:2/tcp/30133".parse::<Multiaddr>()
let multiaddr2 = "/ip6/2002:db8:0:0:0:0:0:2/tcp/30133"
.parse::<Multiaddr>()
.unwrap()
.with(Protocol::P2p(peer_id.into()));
TestMultiaddrsSamePeerCombo(multiaddr1, multiaddr2)
@@ -219,11 +239,13 @@ mod tests {
cache.retain_ids(&vec![first.0, second.0]);
assert_eq!(
None, cache.get_addresses_by_authority_id(&third.0),
None,
cache.get_addresses_by_authority_id(&third.0),
"Expect `get_addresses_by_authority_id` to not return `None` for third authority."
);
assert_eq!(
None, cache.get_authority_id_by_peer_id(&peer_id_from_multiaddr(&third.1).unwrap()),
None,
cache.get_authority_id_by_peer_id(&peer_id_from_multiaddr(&third.1).unwrap()),
"Expect `get_authority_id_by_peer_id` to return `None` for third authority."
);
@@ -253,7 +275,10 @@ mod tests {
let mut cache = AddrCache::new();
cache.insert(authority1.clone(), vec![multiaddr1.clone()]);
cache.insert(authority1.clone(), vec![multiaddr2.clone(), multiaddr3.clone(), multiaddr4.clone()]);
cache.insert(
authority1.clone(),
vec![multiaddr2.clone(), multiaddr3.clone(), multiaddr4.clone()],
);
assert_eq!(
None,
@@ -18,21 +18,26 @@
use crate::worker::schema;
use std::{sync::{Arc, Mutex}, task::Poll};
use std::{
sync::{Arc, Mutex},
task::Poll,
};
use async_trait::async_trait;
use futures::channel::mpsc::{self, channel};
use futures::executor::{block_on, LocalPool};
use futures::future::FutureExt;
use futures::sink::SinkExt;
use futures::task::LocalSpawn;
use libp2p::{kad, core::multiaddr, PeerId};
use futures::{
channel::mpsc::{self, channel},
executor::{block_on, LocalPool},
future::FutureExt,
sink::SinkExt,
task::LocalSpawn,
};
use libp2p::{core::multiaddr, kad, PeerId};
use prometheus_endpoint::prometheus::default_registry;
use sp_api::{ProvideRuntimeApi, ApiRef};
use sp_api::{ApiRef, ProvideRuntimeApi};
use sp_core::crypto::Public;
use sp_keystore::{testing::KeyStore, CryptoStore};
use sp_runtime::traits::{Zero, Block as BlockT, NumberFor};
use sp_runtime::traits::{Block as BlockT, NumberFor, Zero};
use substrate_test_runtime_client::runtime::Block;
use super::*;
@@ -46,9 +51,7 @@ impl ProvideRuntimeApi<Block> for TestApi {
type Api = RuntimeApi;
fn runtime_api<'a>(&'a self) -> ApiRef<'a, Self::Api> {
RuntimeApi {
authorities: self.authorities.clone(),
}.into()
RuntimeApi { authorities: self.authorities.clone() }.into()
}
}
@@ -135,10 +138,7 @@ impl Default for TestNetwork {
let (tx, rx) = mpsc::unbounded();
TestNetwork {
peer_id: PeerId::random(),
external_addresses: vec![
"/ip6/2001:db8::/tcp/30333"
.parse().unwrap(),
],
external_addresses: vec!["/ip6/2001:db8::/tcp/30333".parse().unwrap()],
put_value_call: Default::default(),
get_value_call: Default::default(),
event_sender: tx,
@@ -151,11 +151,17 @@ impl Default for TestNetwork {
impl NetworkProvider for TestNetwork {
fn put_value(&self, key: kad::record::Key, value: Vec<u8>) {
self.put_value_call.lock().unwrap().push((key.clone(), value.clone()));
self.event_sender.clone().unbounded_send(TestNetworkEvent::PutCalled(key, value)).unwrap();
self.event_sender
.clone()
.unbounded_send(TestNetworkEvent::PutCalled(key, value))
.unwrap();
}
fn get_value(&self, key: &kad::record::Key) {
self.get_value_call.lock().unwrap().push(key.clone());
self.event_sender.clone().unbounded_send(TestNetworkEvent::GetCalled(key.clone())).unwrap();
self.event_sender
.clone()
.unbounded_send(TestNetworkEvent::GetCalled(key.clone()))
.unwrap();
}
}
@@ -175,9 +181,8 @@ async fn build_dht_event(
key_store: &KeyStore,
) -> (libp2p::kad::record::Key, Vec<u8>) {
let mut serialized_addresses = vec![];
schema::AuthorityAddresses {
addresses: addresses.into_iter().map(|a| a.to_vec()).collect()
}.encode(&mut serialized_addresses)
schema::AuthorityAddresses { addresses: addresses.into_iter().map(|a| a.to_vec()).collect() }
.encode(&mut serialized_addresses)
.map_err(Error::EncodingProto)
.unwrap();
@@ -192,11 +197,9 @@ async fn build_dht_event(
.unwrap();
let mut signed_addresses = vec![];
schema::SignedAuthorityAddresses {
addresses: serialized_addresses.clone(),
signature,
}
.encode(&mut signed_addresses).unwrap();
schema::SignedAuthorityAddresses { addresses: serialized_addresses.clone(), signature }
.encode(&mut signed_addresses)
.unwrap();
let key = hash_authority_id(&public_key.to_raw_vec());
let value = signed_addresses;
@@ -208,9 +211,7 @@ fn new_registers_metrics() {
let (_dht_event_tx, dht_event_rx) = mpsc::channel(1000);
let network: Arc<TestNetwork> = Arc::new(Default::default());
let key_store = KeyStore::new();
let test_api = Arc::new(TestApi {
authorities: vec![],
});
let test_api = Arc::new(TestApi { authorities: vec![] });
let registry = prometheus_endpoint::Registry::new();
@@ -275,65 +276,67 @@ fn publish_discover_cycle() {
let key_store = KeyStore::new();
let _ = pool.spawner().spawn_local_obj(async move {
let node_a_public = key_store
.sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None)
.await
.unwrap();
let test_api = Arc::new(TestApi {
authorities: vec![node_a_public.into()],
});
let _ = pool.spawner().spawn_local_obj(
async move {
let node_a_public = key_store
.sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None)
.await
.unwrap();
let test_api = Arc::new(TestApi { authorities: vec![node_a_public.into()] });
let (_to_worker, from_service) = mpsc::channel(0);
let mut worker = Worker::new(
from_service,
test_api,
network.clone(),
Box::pin(dht_event_rx),
Role::PublishAndDiscover(key_store.into()),
None,
Default::default(),
);
let (_to_worker, from_service) = mpsc::channel(0);
let mut worker = Worker::new(
from_service,
test_api,
network.clone(),
Box::pin(dht_event_rx),
Role::PublishAndDiscover(key_store.into()),
None,
Default::default(),
);
worker.publish_ext_addresses(false).await.unwrap();
worker.publish_ext_addresses(false).await.unwrap();
// Expect authority discovery to put a new record onto the dht.
assert_eq!(network.put_value_call.lock().unwrap().len(), 1);
// Expect authority discovery to put a new record onto the dht.
assert_eq!(network.put_value_call.lock().unwrap().len(), 1);
let dht_event = {
let (key, value) = network.put_value_call.lock().unwrap().pop().unwrap();
sc_network::DhtEvent::ValueFound(vec![(key, value)])
};
let dht_event = {
let (key, value) = network.put_value_call.lock().unwrap().pop().unwrap();
sc_network::DhtEvent::ValueFound(vec![(key, value)])
};
// Node B discovering node A's address.
// Node B discovering node A's address.
let (mut dht_event_tx, dht_event_rx) = channel(1000);
let test_api = Arc::new(TestApi {
// Make sure node B identifies node A as an authority.
authorities: vec![node_a_public.into()],
});
let network: Arc<TestNetwork> = Arc::new(Default::default());
let key_store = KeyStore::new();
let (mut dht_event_tx, dht_event_rx) = channel(1000);
let test_api = Arc::new(TestApi {
// Make sure node B identifies node A as an authority.
authorities: vec![node_a_public.into()],
});
let network: Arc<TestNetwork> = Arc::new(Default::default());
let key_store = KeyStore::new();
let (_to_worker, from_service) = mpsc::channel(0);
let mut worker = Worker::new(
from_service,
test_api,
network.clone(),
Box::pin(dht_event_rx),
Role::PublishAndDiscover(key_store.into()),
None,
Default::default(),
);
let (_to_worker, from_service) = mpsc::channel(0);
let mut worker = Worker::new(
from_service,
test_api,
network.clone(),
Box::pin(dht_event_rx),
Role::PublishAndDiscover(key_store.into()),
None,
Default::default(),
);
dht_event_tx.try_send(dht_event.clone()).unwrap();
dht_event_tx.try_send(dht_event.clone()).unwrap();
worker.refill_pending_lookups_queue().await.unwrap();
worker.start_new_lookups();
worker.refill_pending_lookups_queue().await.unwrap();
worker.start_new_lookups();
// Make authority discovery handle the event.
worker.handle_dht_event(dht_event).await;
}.boxed_local().into());
// Make authority discovery handle the event.
worker.handle_dht_event(dht_event).await;
}
.boxed_local()
.into(),
);
pool.run();
}
@@ -345,9 +348,7 @@ fn terminate_when_event_stream_terminates() {
let (dht_event_tx, dht_event_rx) = channel(1000);
let network: Arc<TestNetwork> = Arc::new(Default::default());
let key_store = KeyStore::new();
let test_api = Arc::new(TestApi {
authorities: vec![],
});
let test_api = Arc::new(TestApi { authorities: vec![] });
let (to_worker, from_service) = mpsc::channel(0);
let worker = Worker::new(
@@ -358,7 +359,8 @@ fn terminate_when_event_stream_terminates() {
Role::PublishAndDiscover(key_store.into()),
None,
Default::default(),
).run();
)
.run();
futures::pin_mut!(worker);
block_on(async {
@@ -367,7 +369,8 @@ fn terminate_when_event_stream_terminates() {
// Drop sender side of service channel.
drop(to_worker);
assert_eq!(
Poll::Pending, futures::poll!(&mut worker),
Poll::Pending,
futures::poll!(&mut worker),
"Expect the authority discovery module not to terminate once the \
sender side of the service channel is closed.",
);
@@ -377,7 +380,8 @@ fn terminate_when_event_stream_terminates() {
drop(dht_event_tx);
assert_eq!(
Poll::Ready(()), futures::poll!(&mut worker),
Poll::Ready(()),
futures::poll!(&mut worker),
"Expect the authority discovery module to terminate once the \
sending side of the dht event channel is closed.",
);
@@ -390,14 +394,13 @@ fn dont_stop_polling_dht_event_stream_after_bogus_event() {
let peer_id = PeerId::random();
let address: Multiaddr = "/ip6/2001:db8:0:0:0:0:0:1/tcp/30333".parse().unwrap();
address.with(multiaddr::Protocol::P2p(
peer_id.into(),
))
address.with(multiaddr::Protocol::P2p(peer_id.into()))
};
let remote_key_store = KeyStore::new();
let remote_public_key: AuthorityId = block_on(
remote_key_store.sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None),
).unwrap().into();
let remote_public_key: AuthorityId =
block_on(remote_key_store.sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None))
.unwrap()
.into();
let (mut dht_event_tx, dht_event_rx) = channel(1);
let (network, mut network_events) = {
@@ -407,9 +410,7 @@ fn dont_stop_polling_dht_event_stream_after_bogus_event() {
};
let key_store = KeyStore::new();
let test_api = Arc::new(TestApi {
authorities: vec![remote_public_key.clone()],
});
let test_api = Arc::new(TestApi { authorities: vec![remote_public_key.clone()] });
let mut pool = LocalPool::new();
let (mut to_worker, from_service) = mpsc::channel(1);
@@ -427,30 +428,35 @@ fn dont_stop_polling_dht_event_stream_after_bogus_event() {
//
// As this is a local pool, only one future at a time will have the CPU and
// can make progress until the future returns `Pending`.
let _ = pool.spawner().spawn_local_obj(async move {
// Refilling `pending_lookups` only happens every X minutes. Fast
// forward by calling `refill_pending_lookups_queue` directly.
worker.refill_pending_lookups_queue().await.unwrap();
worker.run().await
}.boxed_local().into());
let _ = pool.spawner().spawn_local_obj(
async move {
// Refilling `pending_lookups` only happens every X minutes. Fast
// forward by calling `refill_pending_lookups_queue` directly.
worker.refill_pending_lookups_queue().await.unwrap();
worker.run().await
}
.boxed_local()
.into(),
);
pool.run_until(async {
// Assert worker to trigger a lookup for the one and only authority.
assert!(matches!(
network_events.next().await,
Some(TestNetworkEvent::GetCalled(_))
));
assert!(matches!(network_events.next().await, Some(TestNetworkEvent::GetCalled(_))));
// Send an event that should generate an error
dht_event_tx.send(DhtEvent::ValueFound(Default::default())).await
dht_event_tx
.send(DhtEvent::ValueFound(Default::default()))
.await
.expect("Channel has capacity of 1.");
// Make previously triggered lookup succeed.
let dht_event = {
let (key, value) = build_dht_event(
vec![remote_multiaddr.clone()],
remote_public_key.clone(), &remote_key_store,
).await;
remote_public_key.clone(),
&remote_key_store,
)
.await;
sc_network::DhtEvent::ValueFound(vec![(key, value)])
};
dht_event_tx.send(dht_event).await.expect("Channel has capacity of 1.");
@@ -458,10 +464,10 @@ fn dont_stop_polling_dht_event_stream_after_bogus_event() {
// Expect authority discovery to function normally, now knowing the
// address for the remote node.
let (sender, addresses) = futures::channel::oneshot::channel();
to_worker.send(ServicetoWorkerMsg::GetAddressesByAuthorityId(
remote_public_key,
sender,
)).await.expect("Channel has capacity of 1.");
to_worker
.send(ServicetoWorkerMsg::GetAddressesByAuthorityId(remote_public_key, sender))
.await
.expect("Channel has capacity of 1.");
assert_eq!(Some(vec![remote_multiaddr]), addresses.await.unwrap());
});
}
@@ -469,23 +475,19 @@ fn dont_stop_polling_dht_event_stream_after_bogus_event() {
#[test]
fn limit_number_of_addresses_added_to_cache_per_authority() {
let remote_key_store = KeyStore::new();
let remote_public = block_on(remote_key_store
.sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None))
.unwrap();
let remote_public =
block_on(remote_key_store.sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None))
.unwrap();
let addresses = (0..100).map(|_| {
let peer_id = PeerId::random();
let address: Multiaddr = "/ip6/2001:db8:0:0:0:0:0:1/tcp/30333".parse().unwrap();
address.with(multiaddr::Protocol::P2p(
peer_id.into(),
))
}).collect();
let addresses = (0..100)
.map(|_| {
let peer_id = PeerId::random();
let address: Multiaddr = "/ip6/2001:db8:0:0:0:0:0:1/tcp/30333".parse().unwrap();
address.with(multiaddr::Protocol::P2p(peer_id.into()))
})
.collect();
let dht_event = block_on(build_dht_event(
addresses,
remote_public.into(),
&remote_key_store,
));
let dht_event = block_on(build_dht_event(addresses, remote_public.into(), &remote_key_store));
let (_dht_event_tx, dht_event_rx) = channel(1);
@@ -506,16 +508,20 @@ fn limit_number_of_addresses_added_to_cache_per_authority() {
worker.handle_dht_value_found_event(vec![dht_event]).unwrap();
assert_eq!(
MAX_ADDRESSES_PER_AUTHORITY,
worker.addr_cache.get_addresses_by_authority_id(&remote_public.into()).unwrap().len(),
worker
.addr_cache
.get_addresses_by_authority_id(&remote_public.into())
.unwrap()
.len(),
);
}
#[test]
fn do_not_cache_addresses_without_peer_id() {
let remote_key_store = KeyStore::new();
let remote_public = block_on(remote_key_store
.sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None))
.unwrap();
let remote_public =
block_on(remote_key_store.sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None))
.unwrap();
let multiaddr_with_peer_id = {
let peer_id = PeerId::random();
@@ -524,21 +530,17 @@ fn do_not_cache_addresses_without_peer_id() {
address.with(multiaddr::Protocol::P2p(peer_id.into()))
};
let multiaddr_without_peer_id: Multiaddr = "/ip6/2001:db8:0:0:0:0:0:1/tcp/30333".parse().unwrap();
let multiaddr_without_peer_id: Multiaddr =
"/ip6/2001:db8:0:0:0:0:0:1/tcp/30333".parse().unwrap();
let dht_event = block_on(build_dht_event(
vec![
multiaddr_with_peer_id.clone(),
multiaddr_without_peer_id,
],
vec![multiaddr_with_peer_id.clone(), multiaddr_without_peer_id],
remote_public.into(),
&remote_key_store,
));
let (_dht_event_tx, dht_event_rx) = channel(1);
let local_test_api = Arc::new(TestApi {
authorities: vec![remote_public.into()],
});
let local_test_api = Arc::new(TestApi { authorities: vec![remote_public.into()] });
let local_network: Arc<TestNetwork> = Arc::new(Default::default());
let local_key_store = KeyStore::new();
@@ -578,9 +580,7 @@ fn addresses_to_publish_adds_p2p() {
let (_to_worker, from_service) = mpsc::channel(0);
let worker = Worker::new(
from_service,
Arc::new(TestApi {
authorities: vec![],
}),
Arc::new(TestApi { authorities: vec![] }),
network.clone(),
Box::pin(dht_event_rx),
Role::PublishAndDiscover(Arc::new(KeyStore::new())),
@@ -605,17 +605,16 @@ fn addresses_to_publish_respects_existing_p2p_protocol() {
let network: Arc<TestNetwork> = Arc::new(TestNetwork {
external_addresses: vec![
"/ip6/2001:db8::/tcp/30333/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC"
.parse().unwrap(),
.parse()
.unwrap(),
],
.. Default::default()
..Default::default()
});
let (_to_worker, from_service) = mpsc::channel(0);
let worker = Worker::new(
from_service,
Arc::new(TestApi {
authorities: vec![],
}),
Arc::new(TestApi { authorities: vec![] }),
network.clone(),
Box::pin(dht_event_rx),
Role::PublishAndDiscover(Arc::new(KeyStore::new())),
@@ -624,7 +623,8 @@ fn addresses_to_publish_respects_existing_p2p_protocol() {
);
assert_eq!(
network.external_addresses, worker.addresses_to_publish().collect::<Vec<_>>(),
network.external_addresses,
worker.addresses_to_publish().collect::<Vec<_>>(),
"Expected Multiaddr from `TestNetwork` to not be altered.",
);
}
@@ -635,21 +635,21 @@ fn lookup_throttling() {
let peer_id = PeerId::random();
let address: Multiaddr = "/ip6/2001:db8:0:0:0:0:0:1/tcp/30333".parse().unwrap();
address.with(multiaddr::Protocol::P2p(
peer_id.into(),
))
address.with(multiaddr::Protocol::P2p(peer_id.into()))
};
let remote_key_store = KeyStore::new();
let remote_public_keys: Vec<AuthorityId> = (0..20).map(|_| {
block_on(remote_key_store
.sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None))
.unwrap().into()
}).collect();
let remote_hash_to_key = remote_public_keys.iter()
let remote_public_keys: Vec<AuthorityId> = (0..20)
.map(|_| {
block_on(remote_key_store.sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None))
.unwrap()
.into()
})
.collect();
let remote_hash_to_key = remote_public_keys
.iter()
.map(|k| (hash_authority_id(k.as_ref()), k.clone()))
.collect::<HashMap<_, _>>();
let (mut dht_event_tx, dht_event_rx) = channel(1);
let (_to_worker, from_service) = mpsc::channel(0);
let mut network = TestNetwork::default();
@@ -668,56 +668,61 @@ fn lookup_throttling() {
let mut pool = LocalPool::new();
let metrics = worker.metrics.clone().unwrap();
let _ = pool.spawner().spawn_local_obj(async move {
// Refilling `pending_lookups` only happens every X minutes. Fast
// forward by calling `refill_pending_lookups_queue` directly.
worker.refill_pending_lookups_queue().await.unwrap();
worker.run().await
}.boxed_local().into());
pool.run_until(async {
// Assert worker to trigger MAX_IN_FLIGHT_LOOKUPS lookups.
for _ in 0..MAX_IN_FLIGHT_LOOKUPS {
assert!(matches!(receiver.next().await, Some(TestNetworkEvent::GetCalled(_))));
let _ = pool.spawner().spawn_local_obj(
async move {
// Refilling `pending_lookups` only happens every X minutes. Fast
// forward by calling `refill_pending_lookups_queue` directly.
worker.refill_pending_lookups_queue().await.unwrap();
worker.run().await
}
assert_eq!(
metrics.requests_pending.get(),
(remote_public_keys.len() - MAX_IN_FLIGHT_LOOKUPS) as u64
);
assert_eq!(network.get_value_call.lock().unwrap().len(), MAX_IN_FLIGHT_LOOKUPS);
.boxed_local()
.into(),
);
// Make first lookup succeed.
let remote_hash = network.get_value_call.lock().unwrap().pop().unwrap();
let remote_key: AuthorityId = remote_hash_to_key.get(&remote_hash).unwrap().clone();
let dht_event = {
let (key, value) = build_dht_event(
vec![remote_multiaddr.clone()],
remote_key,
&remote_key_store
).await;
sc_network::DhtEvent::ValueFound(vec![(key, value)])
};
dht_event_tx.send(dht_event).await.expect("Channel has capacity of 1.");
pool.run_until(
async {
// Assert worker to trigger MAX_IN_FLIGHT_LOOKUPS lookups.
for _ in 0..MAX_IN_FLIGHT_LOOKUPS {
assert!(matches!(receiver.next().await, Some(TestNetworkEvent::GetCalled(_))));
}
assert_eq!(
metrics.requests_pending.get(),
(remote_public_keys.len() - MAX_IN_FLIGHT_LOOKUPS) as u64
);
assert_eq!(network.get_value_call.lock().unwrap().len(), MAX_IN_FLIGHT_LOOKUPS);
// Assert worker to trigger another lookup.
assert!(matches!(receiver.next().await, Some(TestNetworkEvent::GetCalled(_))));
assert_eq!(
metrics.requests_pending.get(),
(remote_public_keys.len() - MAX_IN_FLIGHT_LOOKUPS - 1) as u64
);
assert_eq!(network.get_value_call.lock().unwrap().len(), MAX_IN_FLIGHT_LOOKUPS);
// Make first lookup succeed.
let remote_hash = network.get_value_call.lock().unwrap().pop().unwrap();
let remote_key: AuthorityId = remote_hash_to_key.get(&remote_hash).unwrap().clone();
let dht_event = {
let (key, value) =
build_dht_event(vec![remote_multiaddr.clone()], remote_key, &remote_key_store)
.await;
sc_network::DhtEvent::ValueFound(vec![(key, value)])
};
dht_event_tx.send(dht_event).await.expect("Channel has capacity of 1.");
// Make second one fail.
let remote_hash = network.get_value_call.lock().unwrap().pop().unwrap();
let dht_event = sc_network::DhtEvent::ValueNotFound(remote_hash);
dht_event_tx.send(dht_event).await.expect("Channel has capacity of 1.");
// Assert worker to trigger another lookup.
assert!(matches!(receiver.next().await, Some(TestNetworkEvent::GetCalled(_))));
assert_eq!(
metrics.requests_pending.get(),
(remote_public_keys.len() - MAX_IN_FLIGHT_LOOKUPS - 1) as u64
);
assert_eq!(network.get_value_call.lock().unwrap().len(), MAX_IN_FLIGHT_LOOKUPS);
// Assert worker to trigger another lookup.
assert!(matches!(receiver.next().await, Some(TestNetworkEvent::GetCalled(_))));
assert_eq!(
metrics.requests_pending.get(),
(remote_public_keys.len() - MAX_IN_FLIGHT_LOOKUPS - 2) as u64
);
assert_eq!(network.get_value_call.lock().unwrap().len(), MAX_IN_FLIGHT_LOOKUPS);
}.boxed_local());
// Make second one fail.
let remote_hash = network.get_value_call.lock().unwrap().pop().unwrap();
let dht_event = sc_network::DhtEvent::ValueNotFound(remote_hash);
dht_event_tx.send(dht_event).await.expect("Channel has capacity of 1.");
// Assert worker to trigger another lookup.
assert!(matches!(receiver.next().await, Some(TestNetworkEvent::GetCalled(_))));
assert_eq!(
metrics.requests_pending.get(),
(remote_public_keys.len() - MAX_IN_FLIGHT_LOOKUPS - 2) as u64
);
assert_eq!(network.get_value_call.lock().unwrap().len(), MAX_IN_FLIGHT_LOOKUPS);
}
.boxed_local(),
);
}
@@ -20,24 +20,30 @@
// FIXME #1021 move this into sp-consensus
use std::{pin::Pin, time, sync::Arc};
use sc_client_api::backend;
use codec::{Decode, Encode};
use sp_consensus::{evaluation, Proposal, ProofRecording, DisableProofRecording, EnableProofRecording};
use futures::{
channel::oneshot,
future,
future::{Future, FutureExt},
select,
};
use log::{debug, error, info, trace, warn};
use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider};
use sc_client_api::backend;
use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_INFO};
use sc_transaction_pool_api::{InPoolTransaction, TransactionPool};
use sp_api::{ApiExt, ProvideRuntimeApi};
use sp_blockchain::{ApplyExtrinsicFailed::Validity, Error::ApplyExtrinsicFailed, HeaderBackend};
use sp_consensus::{
evaluation, DisableProofRecording, EnableProofRecording, ProofRecording, Proposal,
};
use sp_core::traits::SpawnNamed;
use sp_inherents::InherentData;
use log::{error, info, debug, trace, warn};
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, Hash as HashT, Header as HeaderT, DigestFor, BlakeTwo256},
traits::{BlakeTwo256, Block as BlockT, DigestFor, Hash as HashT, Header as HeaderT},
};
use sc_transaction_pool_api::{TransactionPool, InPoolTransaction};
use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_INFO};
use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider};
use sp_api::{ProvideRuntimeApi, ApiExt};
use futures::{future, future::{Future, FutureExt}, channel::oneshot, select};
use sp_blockchain::{HeaderBackend, ApplyExtrinsicFailed::Validity, Error::ApplyExtrinsicFailed};
use std::marker::PhantomData;
use std::{marker::PhantomData, pin::Pin, sync::Arc, time};
use prometheus_endpoint::Registry as PrometheusRegistry;
use sc_proposer_metrics::MetricsLink as PrometheusMetrics;
@@ -141,14 +147,18 @@ impl<A, B, C, PR> ProposerFactory<A, B, C, PR> {
}
impl<B, Block, C, A, PR> ProposerFactory<A, B, C, PR>
where
A: TransactionPool<Block = Block> + 'static,
B: backend::Backend<Block> + Send + Sync + 'static,
Block: BlockT,
C: BlockBuilderProvider<B, Block, C> + HeaderBackend<Block> + ProvideRuntimeApi<Block>
+ Send + Sync + 'static,
C::Api: ApiExt<Block, StateBackend = backend::StateBackendFor<B, Block>>
+ BlockBuilderApi<Block>,
where
A: TransactionPool<Block = Block> + 'static,
B: backend::Backend<Block> + Send + Sync + 'static,
Block: BlockT,
C: BlockBuilderProvider<B, Block, C>
+ HeaderBackend<Block>
+ ProvideRuntimeApi<Block>
+ Send
+ Sync
+ 'static,
C::Api:
ApiExt<Block, StateBackend = backend::StateBackendFor<B, Block>> + BlockBuilderApi<Block>,
{
fn init_with_now(
&mut self,
@@ -180,26 +190,26 @@ impl<B, Block, C, A, PR> ProposerFactory<A, B, C, PR>
}
}
impl<A, B, Block, C, PR> sp_consensus::Environment<Block> for
ProposerFactory<A, B, C, PR>
where
A: TransactionPool<Block = Block> + 'static,
B: backend::Backend<Block> + Send + Sync + 'static,
Block: BlockT,
C: BlockBuilderProvider<B, Block, C> + HeaderBackend<Block> + ProvideRuntimeApi<Block>
+ Send + Sync + 'static,
C::Api: ApiExt<Block, StateBackend = backend::StateBackendFor<B, Block>>
+ BlockBuilderApi<Block>,
PR: ProofRecording,
impl<A, B, Block, C, PR> sp_consensus::Environment<Block> for ProposerFactory<A, B, C, PR>
where
A: TransactionPool<Block = Block> + 'static,
B: backend::Backend<Block> + Send + Sync + 'static,
Block: BlockT,
C: BlockBuilderProvider<B, Block, C>
+ HeaderBackend<Block>
+ ProvideRuntimeApi<Block>
+ Send
+ Sync
+ 'static,
C::Api:
ApiExt<Block, StateBackend = backend::StateBackendFor<B, Block>> + BlockBuilderApi<Block>,
PR: ProofRecording,
{
type CreateProposer = future::Ready<Result<Self::Proposer, Self::Error>>;
type Proposer = Proposer<B, Block, C, A, PR>;
type Error = sp_blockchain::Error;
fn init(
&mut self,
parent_header: &<Block as BlockT>::Header,
) -> Self::CreateProposer {
fn init(&mut self, parent_header: &<Block as BlockT>::Header) -> Self::CreateProposer {
future::ready(Ok(self.init_with_now(parent_header, Box::new(time::Instant::now))))
}
}
@@ -220,22 +230,28 @@ pub struct Proposer<B, Block: BlockT, C, A: TransactionPool, PR> {
_phantom: PhantomData<(B, PR)>,
}
impl<A, B, Block, C, PR> sp_consensus::Proposer<Block> for
Proposer<B, Block, C, A, PR>
where
A: TransactionPool<Block = Block> + 'static,
B: backend::Backend<Block> + Send + Sync + 'static,
Block: BlockT,
C: BlockBuilderProvider<B, Block, C> + HeaderBackend<Block> + ProvideRuntimeApi<Block>
+ Send + Sync + 'static,
C::Api: ApiExt<Block, StateBackend = backend::StateBackendFor<B, Block>>
+ BlockBuilderApi<Block>,
PR: ProofRecording,
impl<A, B, Block, C, PR> sp_consensus::Proposer<Block> for Proposer<B, Block, C, A, PR>
where
A: TransactionPool<Block = Block> + 'static,
B: backend::Backend<Block> + Send + Sync + 'static,
Block: BlockT,
C: BlockBuilderProvider<B, Block, C>
+ HeaderBackend<Block>
+ ProvideRuntimeApi<Block>
+ Send
+ Sync
+ 'static,
C::Api:
ApiExt<Block, StateBackend = backend::StateBackendFor<B, Block>> + BlockBuilderApi<Block>,
PR: ProofRecording,
{
type Transaction = backend::TransactionFor<B, Block>;
type Proposal = Pin<Box<dyn Future<
Output = Result<Proposal<Block, Self::Transaction, PR::Proof>, Self::Error>
> + Send>>;
type Proposal = Pin<
Box<
dyn Future<Output = Result<Proposal<Block, Self::Transaction, PR::Proof>, Self::Error>>
+ Send,
>,
>;
type Error = sp_blockchain::Error;
type ProofRecording = PR;
type Proof = PR::Proof;
@@ -250,36 +266,38 @@ impl<A, B, Block, C, PR> sp_consensus::Proposer<Block> for
let (tx, rx) = oneshot::channel();
let spawn_handle = self.spawn_handle.clone();
spawn_handle.spawn_blocking("basic-authorship-proposer", Box::pin(async move {
// leave some time for evaluation and block finalization (33%)
let deadline = (self.now)() + max_duration - max_duration / 3;
let res = self.propose_with(
inherent_data,
inherent_digests,
deadline,
block_size_limit,
).await;
if tx.send(res).is_err() {
trace!("Could not send block production result to proposer!");
}
}));
spawn_handle.spawn_blocking(
"basic-authorship-proposer",
Box::pin(async move {
// leave some time for evaluation and block finalization (33%)
let deadline = (self.now)() + max_duration - max_duration / 3;
let res = self
.propose_with(inherent_data, inherent_digests, deadline, block_size_limit)
.await;
if tx.send(res).is_err() {
trace!("Could not send block production result to proposer!");
}
}),
);
async move {
rx.await?
}.boxed()
async move { rx.await? }.boxed()
}
}
impl<A, B, Block, C, PR> Proposer<B, Block, C, A, PR>
where
A: TransactionPool<Block = Block>,
B: backend::Backend<Block> + Send + Sync + 'static,
Block: BlockT,
C: BlockBuilderProvider<B, Block, C> + HeaderBackend<Block> + ProvideRuntimeApi<Block>
+ Send + Sync + 'static,
C::Api: ApiExt<Block, StateBackend = backend::StateBackendFor<B, Block>>
+ BlockBuilderApi<Block>,
PR: ProofRecording,
where
A: TransactionPool<Block = Block>,
B: backend::Backend<Block> + Send + Sync + 'static,
Block: BlockT,
C: BlockBuilderProvider<B, Block, C>
+ HeaderBackend<Block>
+ ProvideRuntimeApi<Block>
+ Send
+ Sync
+ 'static,
C::Api:
ApiExt<Block, StateBackend = backend::StateBackendFor<B, Block>> + BlockBuilderApi<Block>,
PR: ProofRecording,
{
async fn propose_with(
self,
@@ -287,30 +305,30 @@ impl<A, B, Block, C, PR> Proposer<B, Block, C, A, PR>
inherent_digests: DigestFor<Block>,
deadline: time::Instant,
block_size_limit: Option<usize>,
) -> Result<Proposal<Block, backend::TransactionFor<B, Block>, PR::Proof>, sp_blockchain::Error> {
) -> Result<Proposal<Block, backend::TransactionFor<B, Block>, PR::Proof>, sp_blockchain::Error>
{
/// If the block is full we will attempt to push at most
/// this number of transactions before quitting for real.
/// It allows us to increase block utilization.
const MAX_SKIPPED_TRANSACTIONS: usize = 8;
let mut block_builder = self.client.new_block_at(
&self.parent_id,
inherent_digests,
PR::ENABLED,
)?;
let mut block_builder =
self.client.new_block_at(&self.parent_id, inherent_digests, PR::ENABLED)?;
for inherent in block_builder.create_inherents(inherent_data)? {
match block_builder.push(inherent) {
Err(ApplyExtrinsicFailed(Validity(e))) if e.exhausted_resources() =>
warn!("⚠️ Dropping non-mandatory inherent from overweight block."),
Err(ApplyExtrinsicFailed(Validity(e))) if e.was_mandatory() => {
error!("❌️ Mandatory inherent extrinsic returned error. Block cannot be produced.");
error!(
"❌️ Mandatory inherent extrinsic returned error. Block cannot be produced."
);
Err(ApplyExtrinsicFailed(Validity(e)))?
}
},
Err(e) => {
warn!("❗️ Inherent extrinsic returned unexpected error: {}. Dropping.", e);
}
Ok(_) => {}
},
Ok(_) => {},
}
}
@@ -320,9 +338,8 @@ impl<A, B, Block, C, PR> Proposer<B, Block, C, A, PR>
let mut unqueue_invalid = Vec::new();
let mut t1 = self.transaction_pool.ready_at(self.parent_number).fuse();
let mut t2 = futures_timer::Delay::new(
deadline.saturating_duration_since((self.now)()) / 8,
).fuse();
let mut t2 =
futures_timer::Delay::new(deadline.saturating_duration_since((self.now)()) / 8).fuse();
let pending_iterator = select! {
res = t1 => res,
@@ -349,15 +366,14 @@ impl<A, B, Block, C, PR> Proposer<B, Block, C, A, PR>
"Consensus deadline reached when pushing block transactions, \
proceeding with proposing."
);
break;
break
}
let pending_tx_data = pending_tx.data().clone();
let pending_tx_hash = pending_tx.hash().clone();
let block_size = block_builder.estimate_block_size(
self.include_proof_in_block_size_estimation,
);
let block_size =
block_builder.estimate_block_size(self.include_proof_in_block_size_estimation);
if block_size + pending_tx_data.encoded_size() > block_size_limit {
if skipped < MAX_SKIPPED_TRANSACTIONS {
skipped += 1;
@@ -366,11 +382,11 @@ impl<A, B, Block, C, PR> Proposer<B, Block, C, A, PR>
but will try {} more transactions before quitting.",
MAX_SKIPPED_TRANSACTIONS - skipped,
);
continue;
continue
} else {
debug!("Reached block size limit, proceeding with proposing.");
hit_block_size_limit = true;
break;
break
}
}
@@ -379,9 +395,8 @@ impl<A, B, Block, C, PR> Proposer<B, Block, C, A, PR>
Ok(()) => {
transaction_pushed = true;
debug!("[{:?}] Pushed to the block.", pending_tx_hash);
}
Err(ApplyExtrinsicFailed(Validity(e)))
if e.exhausted_resources() => {
},
Err(ApplyExtrinsicFailed(Validity(e))) if e.exhausted_resources() => {
if skipped < MAX_SKIPPED_TRANSACTIONS {
skipped += 1;
debug!(
@@ -390,20 +405,20 @@ impl<A, B, Block, C, PR> Proposer<B, Block, C, A, PR>
);
} else {
debug!("Block is full, proceed with proposing.");
break;
break
}
}
},
Err(e) if skipped > 0 => {
trace!(
"[{:?}] Ignoring invalid transaction when skipping: {}",
pending_tx_hash,
e
);
}
},
Err(e) => {
debug!("[{:?}] Invalid transaction: {}", pending_tx_hash, e);
unqueue_invalid.push(pending_tx_hash);
}
},
}
}
@@ -418,12 +433,10 @@ impl<A, B, Block, C, PR> Proposer<B, Block, C, A, PR>
let (block, storage_changes, proof) = block_builder.build()?.into_inner();
self.metrics.report(
|metrics| {
metrics.number_of_transactions.set(block.extrinsics().len() as u64);
metrics.block_constructed.observe(block_timer.elapsed().as_secs_f64());
}
);
self.metrics.report(|metrics| {
metrics.number_of_transactions.set(block.extrinsics().len() as u64);
metrics.block_constructed.observe(block_timer.elapsed().as_secs_f64());
});
info!(
"🎁 Prepared block for proposing at {} [hash: {:?}; parent_hash: {}; extrinsics ({}): [{}]]",
@@ -449,16 +462,14 @@ impl<A, B, Block, C, PR> Proposer<B, Block, C, A, PR>
error!("Failed to verify block encoding/decoding");
}
if let Err(err) = evaluation::evaluate_initial(
&block,
&self.parent_hash,
self.parent_number,
) {
if let Err(err) =
evaluation::evaluate_initial(&block, &self.parent_hash, self.parent_number)
{
error!("Failed to evaluate authored block: {:?}", err);
}
let proof = PR::into_proof(proof)
.map_err(|e| sp_blockchain::Error::Application(Box::new(e)))?;
let proof =
PR::into_proof(proof).map_err(|e| sp_blockchain::Error::Application(Box::new(e)))?;
Ok(Proposal { block, proof, storage_changes })
}
}
@@ -467,19 +478,20 @@ impl<A, B, Block, C, PR> Proposer<B, Block, C, A, PR>
mod tests {
use super::*;
use futures::executor::block_on;
use parking_lot::Mutex;
use sp_consensus::{BlockOrigin, Proposer};
use substrate_test_runtime_client::{
prelude::*, TestClientBuilder, runtime::{Extrinsic, Transfer}, TestClientBuilderExt,
};
use sc_transaction_pool_api::{ChainEvent, MaintainedTransactionPool, TransactionSource};
use sc_client_api::Backend;
use sc_transaction_pool::BasicPool;
use sc_transaction_pool_api::{ChainEvent, MaintainedTransactionPool, TransactionSource};
use sp_api::Core;
use sp_blockchain::HeaderBackend;
use sp_consensus::{BlockOrigin, Environment, Proposer};
use sp_runtime::traits::NumberFor;
use sc_client_api::Backend;
use futures::executor::block_on;
use sp_consensus::Environment;
use substrate_test_runtime_client::{
prelude::*,
runtime::{Extrinsic, Transfer},
TestClientBuilder, TestClientBuilderExt,
};
const SOURCE: TransactionSource = TransactionSource::External;
@@ -489,16 +501,15 @@ mod tests {
nonce,
from: AccountKeyring::Alice.into(),
to: Default::default(),
}.into_signed_tx()
}
.into_signed_tx()
}
fn chain_event<B: BlockT>(header: B::Header) -> ChainEvent<B>
where NumberFor<B>: From<u64>
where
NumberFor<B>: From<u64>,
{
ChainEvent::NewBestBlock {
hash: header.hash(),
tree_route: None,
}
ChainEvent::NewBestBlock { hash: header.hash(), tree_route: None }
}
#[test]
@@ -514,25 +525,20 @@ mod tests {
client.clone(),
);
block_on(
txpool.submit_at(&BlockId::number(0), SOURCE, vec![extrinsic(0), extrinsic(1)])
).unwrap();
block_on(txpool.submit_at(&BlockId::number(0), SOURCE, vec![extrinsic(0), extrinsic(1)]))
.unwrap();
block_on(
txpool.maintain(chain_event(
client.header(&BlockId::Number(0u64))
client
.header(&BlockId::Number(0u64))
.expect("header get error")
.expect("there should be header")
))
.expect("there should be header"),
)),
);
let mut proposer_factory = ProposerFactory::new(
spawner.clone(),
client.clone(),
txpool.clone(),
None,
None,
);
let mut proposer_factory =
ProposerFactory::new(spawner.clone(), client.clone(), txpool.clone(), None, None);
let cell = Mutex::new((false, time::Instant::now()));
let proposer = proposer_factory.init_with_now(
@@ -541,20 +547,21 @@ mod tests {
let mut value = cell.lock();
if !value.0 {
value.0 = true;
return value.1;
return value.1
}
let old = value.1;
let new = old + time::Duration::from_secs(2);
*value = (true, new);
old
})
}),
);
// when
let deadline = time::Duration::from_secs(3);
let block = block_on(
proposer.propose(Default::default(), Default::default(), deadline, None)
).map(|r| r.block).unwrap();
let block =
block_on(proposer.propose(Default::default(), Default::default(), deadline, None))
.map(|r| r.block)
.unwrap();
// then
// block should have some extrinsics although we have some more in the pool.
@@ -574,13 +581,8 @@ mod tests {
client.clone(),
);
let mut proposer_factory = ProposerFactory::new(
spawner.clone(),
client.clone(),
txpool.clone(),
None,
None,
);
let mut proposer_factory =
ProposerFactory::new(spawner.clone(), client.clone(), txpool.clone(), None, None);
let cell = Mutex::new((false, time::Instant::now()));
let proposer = proposer_factory.init_with_now(
@@ -589,18 +591,18 @@ mod tests {
let mut value = cell.lock();
if !value.0 {
value.0 = true;
return value.1;
return value.1
}
let new = value.1 + time::Duration::from_secs(160);
*value = (true, new);
new
})
}),
);
let deadline = time::Duration::from_secs(1);
block_on(
proposer.propose(Default::default(), Default::default(), deadline, None)
).map(|r| r.block).unwrap();
block_on(proposer.propose(Default::default(), Default::default(), deadline, None))
.map(|r| r.block)
.unwrap();
}
#[test]
@@ -619,25 +621,19 @@ mod tests {
let genesis_hash = client.info().best_hash;
let block_id = BlockId::Hash(genesis_hash);
block_on(
txpool.submit_at(&BlockId::number(0), SOURCE, vec![extrinsic(0)]),
).unwrap();
block_on(txpool.submit_at(&BlockId::number(0), SOURCE, vec![extrinsic(0)])).unwrap();
block_on(
txpool.maintain(chain_event(
client.header(&BlockId::Number(0u64))
client
.header(&BlockId::Number(0u64))
.expect("header get error")
.expect("there should be header"),
))
)),
);
let mut proposer_factory = ProposerFactory::new(
spawner.clone(),
client.clone(),
txpool.clone(),
None,
None,
);
let mut proposer_factory =
ProposerFactory::new(spawner.clone(), client.clone(), txpool.clone(), None, None);
let proposer = proposer_factory.init_with_now(
&client.header(&block_id).unwrap().unwrap(),
@@ -645,9 +641,9 @@ mod tests {
);
let deadline = time::Duration::from_secs(9);
let proposal = block_on(
proposer.propose(Default::default(), Default::default(), deadline, None),
).unwrap();
let proposal =
block_on(proposer.propose(Default::default(), Default::default(), deadline, None))
.unwrap();
assert_eq!(proposal.block.extrinsics().len(), 1);
@@ -655,16 +651,13 @@ mod tests {
api.execute_block(&block_id, proposal.block).unwrap();
let state = backend.state_at(block_id).unwrap();
let changes_trie_state = backend::changes_tries_state_at_block(
&block_id,
backend.changes_trie_storage(),
).unwrap();
let changes_trie_state =
backend::changes_tries_state_at_block(&block_id, backend.changes_trie_storage())
.unwrap();
let storage_changes = api.into_storage_changes(
&state,
changes_trie_state.as_ref(),
genesis_hash,
).unwrap();
let storage_changes = api
.into_storage_changes(&state, changes_trie_state.as_ref(), genesis_hash)
.unwrap();
assert_eq!(
proposal.storage_changes.transaction_storage_root,
@@ -685,8 +678,10 @@ mod tests {
client.clone(),
);
block_on(
txpool.submit_at(&BlockId::number(0), SOURCE, vec![
block_on(txpool.submit_at(
&BlockId::number(0),
SOURCE,
vec![
extrinsic(0),
extrinsic(1),
Transfer {
@@ -704,22 +699,16 @@ mod tests {
}.into_resources_exhausting_tx(),
extrinsic(5),
extrinsic(6),
])
).unwrap();
],
))
.unwrap();
let mut proposer_factory = ProposerFactory::new(
spawner.clone(),
client.clone(),
txpool.clone(),
None,
None,
);
let mut propose_block = |
client: &TestClient,
number,
expected_block_extrinsics,
expected_pool_transactions,
| {
let mut proposer_factory =
ProposerFactory::new(spawner.clone(), client.clone(), txpool.clone(), None, None);
let mut propose_block = |client: &TestClient,
number,
expected_block_extrinsics,
expected_pool_transactions| {
let proposer = proposer_factory.init_with_now(
&client.header(&BlockId::number(number)).unwrap().unwrap(),
Box::new(move || time::Instant::now()),
@@ -727,9 +716,10 @@ mod tests {
// when
let deadline = time::Duration::from_secs(9);
let block = block_on(
proposer.propose(Default::default(), Default::default(), deadline, None)
).map(|r| r.block).unwrap();
let block =
block_on(proposer.propose(Default::default(), Default::default(), deadline, None))
.map(|r| r.block)
.unwrap();
// then
// block should have some extrinsics although we have some more in the pool.
@@ -741,10 +731,11 @@ mod tests {
block_on(
txpool.maintain(chain_event(
client.header(&BlockId::Number(0u64))
client
.header(&BlockId::Number(0u64))
.expect("header get error")
.expect("there should be header")
))
.expect("there should be header"),
)),
);
// let's create one block and import it
@@ -753,10 +744,11 @@ mod tests {
block_on(
txpool.maintain(chain_event(
client.header(&BlockId::Number(1))
client
.header(&BlockId::Number(1))
.expect("header get error")
.expect("there should be header")
))
.expect("there should be header"),
)),
);
// now let's make sure that we can still make some progress
@@ -775,7 +767,8 @@ mod tests {
spawner.clone(),
client.clone(),
);
let genesis_header = client.header(&BlockId::Number(0u64))
let genesis_header = client
.header(&BlockId::Number(0u64))
.expect("header get error")
.expect("there should be header");
@@ -784,40 +777,43 @@ mod tests {
.map(|v| Extrinsic::IncludeData(vec![v as u8; 10]))
.collect::<Vec<_>>();
let block_limit = genesis_header.encoded_size()
+ extrinsics.iter().take(extrinsics_num - 1).map(Encode::encoded_size).sum::<usize>()
+ Vec::<Extrinsic>::new().encoded_size();
let block_limit = genesis_header.encoded_size() +
extrinsics
.iter()
.take(extrinsics_num - 1)
.map(Encode::encoded_size)
.sum::<usize>() +
Vec::<Extrinsic>::new().encoded_size();
block_on(
txpool.submit_at(&BlockId::number(0), SOURCE, extrinsics)
).unwrap();
block_on(txpool.submit_at(&BlockId::number(0), SOURCE, extrinsics)).unwrap();
block_on(txpool.maintain(chain_event(genesis_header.clone())));
let mut proposer_factory = ProposerFactory::new(
spawner.clone(),
client.clone(),
txpool.clone(),
None,
None,
);
let mut proposer_factory =
ProposerFactory::new(spawner.clone(), client.clone(), txpool.clone(), None, None);
let proposer = block_on(proposer_factory.init(&genesis_header)).unwrap();
// Give it enough time
let deadline = time::Duration::from_secs(300);
let block = block_on(
proposer.propose(Default::default(), Default::default(), deadline, Some(block_limit))
).map(|r| r.block).unwrap();
let block = block_on(proposer.propose(
Default::default(),
Default::default(),
deadline,
Some(block_limit),
))
.map(|r| r.block)
.unwrap();
// Based on the block limit, one transaction shouldn't be included.
assert_eq!(block.extrinsics().len(), extrinsics_num - 1);
let proposer = block_on(proposer_factory.init(&genesis_header)).unwrap();
let block = block_on(
proposer.propose(Default::default(), Default::default(), deadline, None,
)).map(|r| r.block).unwrap();
let block =
block_on(proposer.propose(Default::default(), Default::default(), deadline, None))
.map(|r| r.block)
.unwrap();
// Without a block limit we should include all of them
assert_eq!(block.extrinsics().len(), extrinsics_num);
@@ -833,9 +829,14 @@ mod tests {
let proposer = block_on(proposer_factory.init(&genesis_header)).unwrap();
// Give it enough time
let block = block_on(
proposer.propose(Default::default(), Default::default(), deadline, Some(block_limit))
).map(|r| r.block).unwrap();
let block = block_on(proposer.propose(
Default::default(),
Default::default(),
deadline,
Some(block_limit),
))
.map(|r| r.block)
.unwrap();
// The block limit didn't changed, but we now include the proof in the estimation of the
// block size and thus, one less transaction should fit into the limit.
+7 -8
View File
@@ -41,12 +41,12 @@
//! # );
//! // The first step is to create a `ProposerFactory`.
//! let mut proposer_factory = ProposerFactory::new(
//! spawner,
//! client.clone(),
//! txpool.clone(),
//! None,
//! None,
//! );
//! spawner,
//! client.clone(),
//! txpool.clone(),
//! None,
//! None,
//! );
//!
//! // From this factory, we create a `Proposer`.
//! let proposer = proposer_factory.init(
@@ -69,8 +69,7 @@
//! let block = futures::executor::block_on(future).unwrap();
//! println!("Generated block: {:?}", block.block);
//! ```
//!
mod basic_authorship;
pub use crate::basic_authorship::{ProposerFactory, Proposer, DEFAULT_BLOCK_SIZE_LIMIT};
pub use crate::basic_authorship::{Proposer, ProposerFactory, DEFAULT_BLOCK_SIZE_LIMIT};
+47 -44
View File
@@ -28,14 +28,14 @@
use codec::Encode;
use sp_runtime::{
generic::BlockId,
traits::{Header as HeaderT, Hash, Block as BlockT, HashFor, DigestFor, NumberFor, One},
use sp_api::{
ApiExt, ApiRef, Core, ProvideRuntimeApi, StorageChanges, StorageProof, TransactionOutcome,
};
use sp_blockchain::{ApplyExtrinsicFailed, Error};
use sp_core::ExecutionContext;
use sp_api::{
Core, ApiExt, ApiRef, ProvideRuntimeApi, StorageChanges, StorageProof, TransactionOutcome,
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, DigestFor, Hash, HashFor, Header as HeaderT, NumberFor, One},
};
pub use sp_block_builder::BlockBuilder as BlockBuilderApi;
@@ -94,7 +94,9 @@ pub struct BuiltBlock<Block: BlockT, StateBackend: backend::StateBackend<HashFor
pub proof: Option<StorageProof>,
}
impl<Block: BlockT, StateBackend: backend::StateBackend<HashFor<Block>>> BuiltBlock<Block, StateBackend> {
impl<Block: BlockT, StateBackend: backend::StateBackend<HashFor<Block>>>
BuiltBlock<Block, StateBackend>
{
/// Convert into the inner values.
pub fn into_inner(self) -> (Block, StorageChanges<StateBackend, Block>, Option<StorageProof>) {
(self.block, self.storage_changes, self.proof)
@@ -103,11 +105,11 @@ impl<Block: BlockT, StateBackend: backend::StateBackend<HashFor<Block>>> BuiltBl
/// Block builder provider
pub trait BlockBuilderProvider<B, Block, RA>
where
Block: BlockT,
B: backend::Backend<Block>,
Self: Sized,
RA: ProvideRuntimeApi<Block>,
where
Block: BlockT,
B: backend::Backend<Block>,
Self: Sized,
RA: ProvideRuntimeApi<Block>,
{
/// Create a new block, built on top of `parent`.
///
@@ -143,7 +145,8 @@ impl<'a, Block, A, B> BlockBuilder<'a, Block, A, B>
where
Block: BlockT,
A: ProvideRuntimeApi<Block> + 'a,
A::Api: BlockBuilderApi<Block> + ApiExt<Block, StateBackend = backend::StateBackendFor<B, Block>>,
A::Api:
BlockBuilderApi<Block> + ApiExt<Block, StateBackend = backend::StateBackendFor<B, Block>>,
B: backend::Backend<Block>,
{
/// Create a new instance of builder based on the given `parent_hash` and `parent_number`.
@@ -177,9 +180,7 @@ where
let block_id = BlockId::Hash(parent_hash);
api.initialize_block_with_context(
&block_id, ExecutionContext::BlockConstruction, &header,
)?;
api.initialize_block_with_context(&block_id, ExecutionContext::BlockConstruction, &header)?;
Ok(Self {
parent_hash,
@@ -207,12 +208,10 @@ where
Ok(Ok(_)) => {
extrinsics.push(xt);
TransactionOutcome::Commit(Ok(()))
}
Ok(Err(tx_validity)) => {
TransactionOutcome::Rollback(
Err(ApplyExtrinsicFailed::Validity(tx_validity).into()),
)
},
Ok(Err(tx_validity)) => TransactionOutcome::Rollback(Err(
ApplyExtrinsicFailed::Validity(tx_validity).into(),
)),
Err(e) => TransactionOutcome::Rollback(Err(Error::from(e))),
}
})
@@ -224,9 +223,9 @@ where
/// supplied by `self.api`, combined as [`BuiltBlock`].
/// The storage proof will be `Some(_)` when proof recording was enabled.
pub fn build(mut self) -> Result<BuiltBlock<Block, backend::StateBackendFor<B, Block>>, Error> {
let header = self.api.finalize_block_with_context(
&self.block_id, ExecutionContext::BlockConstruction
)?;
let header = self
.api
.finalize_block_with_context(&self.block_id, ExecutionContext::BlockConstruction)?;
debug_assert_eq!(
header.extrinsics_root().clone(),
@@ -244,11 +243,10 @@ where
)?;
let parent_hash = self.parent_hash;
let storage_changes = self.api.into_storage_changes(
&state,
changes_trie_state.as_ref(),
parent_hash,
).map_err(|e| sp_blockchain::Error::StorageChanges(e))?;
let storage_changes = self
.api
.into_storage_changes(&state, changes_trie_state.as_ref(), parent_hash)
.map_err(|e| sp_blockchain::Error::StorageChanges(e))?;
Ok(BuiltBlock {
block: <Block as BlockT>::new(header, self.extrinsics),
@@ -265,15 +263,17 @@ where
inherent_data: sp_inherents::InherentData,
) -> Result<Vec<Block::Extrinsic>, Error> {
let block_id = self.block_id;
self.api.execute_in_transaction(move |api| {
// `create_inherents` should not change any state, to ensure this we always rollback
// the transaction.
TransactionOutcome::Rollback(api.inherent_extrinsics_with_context(
&block_id,
ExecutionContext::BlockConstruction,
inherent_data
))
}).map_err(|e| Error::Application(Box::new(e)))
self.api
.execute_in_transaction(move |api| {
// `create_inherents` should not change any state, to ensure this we always rollback
// the transaction.
TransactionOutcome::Rollback(api.inherent_extrinsics_with_context(
&block_id,
ExecutionContext::BlockConstruction,
inherent_data,
))
})
.map_err(|e| Error::Application(Box::new(e)))
}
/// Estimate the size of the block in the current state.
@@ -312,19 +312,22 @@ mod tests {
RecordProof::Yes,
Default::default(),
&*backend,
).unwrap().build().unwrap();
)
.unwrap()
.build()
.unwrap();
let proof = block.proof.expect("Proof is build on request");
let backend = sp_state_machine::create_proof_check_backend::<Blake2Hasher>(
block.storage_changes.transaction_storage_root,
proof,
).unwrap();
)
.unwrap();
assert!(
backend.storage(&sp_core::storage::well_known_keys::CODE)
.unwrap_err()
.contains("Database missing expected key"),
);
assert!(backend
.storage(&sp_core::storage::well_known_keys::CODE)
.unwrap_err()
.contains("Database missing expected key"),);
}
}
+35 -38
View File
@@ -17,9 +17,9 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use proc_macro2::{Span, TokenStream};
use quote::quote;
use syn::{DeriveInput, Ident, Error};
use proc_macro_crate::{crate_name, FoundCrate};
use quote::quote;
use syn::{DeriveInput, Error, Ident};
const CRATE_NAME: &str = "sc-chain-spec";
const ATTRIBUTE_NAME: &str = "forks";
@@ -31,14 +31,18 @@ const ATTRIBUTE_NAME: &str = "forks";
pub fn extension_derive(ast: &DeriveInput) -> proc_macro::TokenStream {
derive(ast, |crate_name, name, generics: &syn::Generics, field_names, field_types, fields| {
let (impl_generics, ty_generics, where_clause) = generics.split_for_impl();
let forks = fields.named.iter().find_map(|f| {
if f.attrs.iter().any(|attr| attr.path.is_ident(ATTRIBUTE_NAME)) {
let typ = &f.ty;
Some(quote! { #typ })
} else {
None
}
}).unwrap_or_else(|| quote! { #crate_name::NoExtension });
let forks = fields
.named
.iter()
.find_map(|f| {
if f.attrs.iter().any(|attr| attr.path.is_ident(ATTRIBUTE_NAME)) {
let typ = &f.ty;
Some(quote! { #typ })
} else {
None
}
})
.unwrap_or_else(|| quote! { #crate_name::NoExtension });
quote! {
impl #impl_generics #crate_name::Extension for #name #ty_generics #where_clause {
@@ -80,13 +84,12 @@ pub fn group_derive(ast: &DeriveInput) -> proc_macro::TokenStream {
Ok(FoundCrate::Itself) => Ident::new("serde", Span::call_site()),
Ok(FoundCrate::Name(name)) => Ident::new(&name, Span::call_site()),
Err(e) => {
let err = Error::new(
Span::call_site(),
&format!("Could not find `serde` crate: {}", e),
).to_compile_error();
let err =
Error::new(Span::call_site(), &format!("Could not find `serde` crate: {}", e))
.to_compile_error();
return quote!( #err ).into();
}
return quote!( #err ).into()
},
};
quote! {
@@ -131,14 +134,20 @@ pub fn group_derive(ast: &DeriveInput) -> proc_macro::TokenStream {
pub fn derive(
ast: &DeriveInput,
derive: impl Fn(
&Ident, &Ident, &syn::Generics, Vec<&Ident>, Vec<&syn::Type>, &syn::FieldsNamed,
&Ident,
&Ident,
&syn::Generics,
Vec<&Ident>,
Vec<&syn::Type>,
&syn::FieldsNamed,
) -> TokenStream,
) -> proc_macro::TokenStream {
let err = || {
let err = Error::new(
Span::call_site(),
"ChainSpecGroup is only available for structs with named fields."
).to_compile_error();
"ChainSpecGroup is only available for structs with named fields.",
)
.to_compile_error();
quote!( #err ).into()
};
@@ -168,47 +177,35 @@ pub fn derive(
derive(&crate_name, name, &ast.generics, field_names, field_types, fields).into()
}
fn generate_fork_fields(
crate_name: &Ident,
names: &[&Ident],
types: &[&syn::Type],
) -> TokenStream {
fn generate_fork_fields(crate_name: &Ident, names: &[&Ident], types: &[&syn::Type]) -> TokenStream {
let crate_name = std::iter::repeat(crate_name);
quote! {
#( pub #names: Option<<#types as #crate_name::Group>::Fork>, )*
}
}
fn generate_base_to_fork(
fork_name: &Ident,
names: &[&Ident],
) -> TokenStream {
fn generate_base_to_fork(fork_name: &Ident, names: &[&Ident]) -> TokenStream {
let names2 = names.to_vec();
quote!{
quote! {
#fork_name {
#( #names: Some(self.#names2.to_fork()), )*
}
}
}
fn generate_combine_with(
names: &[&Ident],
) -> TokenStream {
fn generate_combine_with(names: &[&Ident]) -> TokenStream {
let names2 = names.to_vec();
quote!{
quote! {
#( self.#names.combine_with(other.#names2); )*
}
}
fn generate_fork_to_base(
fork: &Ident,
names: &[&Ident],
) -> TokenStream {
fn generate_fork_to_base(fork: &Ident, names: &[&Ident]) -> TokenStream {
let names2 = names.to_vec();
quote!{
quote! {
Some(#fork {
#( #names: self.#names2?.to_base()?, )*
})
+94 -96
View File
@@ -19,15 +19,20 @@
//! Substrate chain configurations.
#![warn(missing_docs)]
use std::{borrow::Cow, fs::File, path::PathBuf, sync::Arc, collections::HashMap};
use serde::{Serialize, Deserialize};
use sp_core::{storage::{StorageKey, StorageData, ChildInfo, Storage, StorageChild}, Bytes};
use sp_runtime::BuildStorage;
use serde_json as json;
use crate::{RuntimeGenesis, ChainType, extension::GetExtension, Properties};
use crate::{extension::GetExtension, ChainType, Properties, RuntimeGenesis};
use sc_network::config::MultiaddrWithPeerId;
use sc_telemetry::TelemetryEndpoints;
use sp_runtime::traits::{Block as BlockT, NumberFor};
use serde::{Deserialize, Serialize};
use serde_json as json;
use sp_core::{
storage::{ChildInfo, Storage, StorageChild, StorageData, StorageKey},
Bytes,
};
use sp_runtime::{
traits::{Block as BlockT, NumberFor},
BuildStorage,
};
use std::{borrow::Cow, collections::HashMap, fs::File, path::PathBuf, sync::Arc};
enum GenesisSource<G> {
File(PathBuf),
@@ -56,8 +61,8 @@ impl<G: RuntimeGenesis> GenesisSource<G> {
match self {
Self::File(path) => {
let file = File::open(path)
.map_err(|e| format!("Error opening spec file: {}", e))?;
let file =
File::open(path).map_err(|e| format!("Error opening spec file: {}", e))?;
let genesis: GenesisContainer<G> = json::from_reader(file)
.map_err(|e| format!("Error parsing spec file: {}", e))?;
Ok(genesis.genesis)
@@ -69,22 +74,25 @@ impl<G: RuntimeGenesis> GenesisSource<G> {
},
Self::Factory(f) => Ok(Genesis::Runtime(f())),
Self::Storage(storage) => {
let top = storage.top
let top = storage
.top
.iter()
.map(|(k, v)| (StorageKey(k.clone()), StorageData(v.clone())))
.collect();
let children_default = storage.children_default
let children_default = storage
.children_default
.iter()
.map(|(k, child)|
(
StorageKey(k.clone()),
child.data
.map(|(k, child)| {
(
StorageKey(k.clone()),
child
.data
.iter()
.map(|(k, v)| (StorageKey(k.clone()), StorageData(v.clone())))
.collect()
)
)
.collect(),
)
})
.collect();
Ok(Genesis::Raw(RawGenesis { top, children_default }))
@@ -99,24 +107,24 @@ impl<G: RuntimeGenesis, E> BuildStorage for ChainSpec<G, E> {
Genesis::Runtime(gc) => gc.build_storage(),
Genesis::Raw(RawGenesis { top: map, children_default: children_map }) => Ok(Storage {
top: map.into_iter().map(|(k, v)| (k.0, v.0)).collect(),
children_default: children_map.into_iter().map(|(storage_key, child_content)| {
let child_info = ChildInfo::new_default(storage_key.0.as_slice());
(
storage_key.0,
StorageChild {
data: child_content.into_iter().map(|(k, v)| (k.0, v.0)).collect(),
child_info,
},
)
}).collect(),
children_default: children_map
.into_iter()
.map(|(storage_key, child_content)| {
let child_info = ChildInfo::new_default(storage_key.0.as_slice());
(
storage_key.0,
StorageChild {
data: child_content.into_iter().map(|(k, v)| (k.0, v.0)).collect(),
child_info,
},
)
})
.collect(),
}),
}
}
fn assimilate_storage(
&self,
_: &mut Storage,
) -> Result<(), String> {
fn assimilate_storage(&self, _: &mut Storage) -> Result<(), String> {
Err("`assimilate_storage` not implemented for `ChainSpec`.".into())
}
}
@@ -181,10 +189,7 @@ pub struct ChainSpec<G, E = NoExtension> {
impl<G, E: Clone> Clone for ChainSpec<G, E> {
fn clone(&self) -> Self {
ChainSpec {
client_spec: self.client_spec.clone(),
genesis: self.genesis.clone(),
}
ChainSpec { client_spec: self.client_spec.clone(), genesis: self.genesis.clone() }
}
}
@@ -258,10 +263,7 @@ impl<G, E> ChainSpec<G, E> {
code_substitutes: HashMap::new(),
};
ChainSpec {
client_spec,
genesis: GenesisSource::Factory(Arc::new(constructor)),
}
ChainSpec { client_spec, genesis: GenesisSource::Factory(Arc::new(constructor)) }
}
/// Type of the chain.
@@ -281,22 +283,15 @@ impl<G, E: serde::de::DeserializeOwned> ChainSpec<G, E> {
let json = json.into();
let client_spec = json::from_slice(json.as_ref())
.map_err(|e| format!("Error parsing spec file: {}", e))?;
Ok(ChainSpec {
client_spec,
genesis: GenesisSource::Binary(json),
})
Ok(ChainSpec { client_spec, genesis: GenesisSource::Binary(json) })
}
/// Parse json file into a `ChainSpec`
pub fn from_json_file(path: PathBuf) -> Result<Self, String> {
let file = File::open(&path)
.map_err(|e| format!("Error opening spec file: {}", e))?;
let client_spec = json::from_reader(file)
.map_err(|e| format!("Error parsing spec file: {}", e))?;
Ok(ChainSpec {
client_spec,
genesis: GenesisSource::File(path),
})
let file = File::open(&path).map_err(|e| format!("Error opening spec file: {}", e))?;
let client_spec =
json::from_reader(file).map_err(|e| format!("Error parsing spec file: {}", e))?;
Ok(ChainSpec { client_spec, genesis: GenesisSource::File(path) })
}
}
@@ -312,33 +307,34 @@ impl<G: RuntimeGenesis, E: serde::Serialize + Clone + 'static> ChainSpec<G, E> {
let genesis = match (raw, self.genesis.resolve()?) {
(true, Genesis::Runtime(g)) => {
let storage = g.build_storage()?;
let top = storage.top.into_iter()
.map(|(k, v)| (StorageKey(k), StorageData(v)))
.collect();
let children_default = storage.children_default.into_iter()
.map(|(sk, child)| (
StorageKey(sk),
child.data.into_iter()
.map(|(k, v)| (StorageKey(k), StorageData(v)))
.collect(),
))
let top =
storage.top.into_iter().map(|(k, v)| (StorageKey(k), StorageData(v))).collect();
let children_default = storage
.children_default
.into_iter()
.map(|(sk, child)| {
(
StorageKey(sk),
child
.data
.into_iter()
.map(|(k, v)| (StorageKey(k), StorageData(v)))
.collect(),
)
})
.collect();
Genesis::Raw(RawGenesis { top, children_default })
},
(_, genesis) => genesis,
};
Ok(JsonContainer {
client_spec: self.client_spec.clone(),
genesis,
})
Ok(JsonContainer { client_spec: self.client_spec.clone(), genesis })
}
/// Dump to json string.
pub fn as_json(&self, raw: bool) -> Result<String, String> {
let container = self.json_container(raw)?;
json::to_string_pretty(&container)
.map_err(|e| format!("Error generating spec json: {}", e))
json::to_string_pretty(&container).map_err(|e| format!("Error generating spec json: {}", e))
}
}
@@ -404,7 +400,11 @@ where
}
fn code_substitutes(&self) -> std::collections::HashMap<String, Vec<u8>> {
self.client_spec.code_substitutes.iter().map(|(h, c)| (h.clone(), c.0.clone())).collect()
self.client_spec
.code_substitutes
.iter()
.map(|(h, c)| (h.clone(), c.0.clone()))
.collect()
}
}
@@ -417,7 +417,8 @@ pub struct LightSyncState<Block: BlockT> {
/// The babe weight of the finalized block.
pub babe_finalized_block_weight: sp_consensus_babe::BabeBlockWeight,
/// The authority set for grandpa.
pub grandpa_authority_set: sc_finality_grandpa::AuthoritySet<<Block as BlockT>::Hash, NumberFor<Block>>,
pub grandpa_authority_set:
sc_finality_grandpa::AuthoritySet<<Block as BlockT>::Hash, NumberFor<Block>>,
}
impl<Block: BlockT> LightSyncState<Block> {
@@ -427,25 +428,25 @@ impl<Block: BlockT> LightSyncState<Block> {
SerializableLightSyncState {
finalized_block_header: StorageData(self.finalized_block_header.encode()),
babe_epoch_changes:
StorageData(self.babe_epoch_changes.encode()),
babe_finalized_block_weight:
self.babe_finalized_block_weight,
grandpa_authority_set:
StorageData(self.grandpa_authority_set.encode()),
babe_epoch_changes: StorageData(self.babe_epoch_changes.encode()),
babe_finalized_block_weight: self.babe_finalized_block_weight,
grandpa_authority_set: StorageData(self.grandpa_authority_set.encode()),
}
}
/// Convert from a `SerializableLightSyncState`.
pub fn from_serializable(serialized: &SerializableLightSyncState) -> Result<Self, codec::Error> {
pub fn from_serializable(
serialized: &SerializableLightSyncState,
) -> Result<Self, codec::Error> {
Ok(Self {
finalized_block_header: codec::Decode::decode(&mut &serialized.finalized_block_header.0[..])?,
babe_epoch_changes:
codec::Decode::decode(&mut &serialized.babe_epoch_changes.0[..])?,
babe_finalized_block_weight:
serialized.babe_finalized_block_weight,
grandpa_authority_set:
codec::Decode::decode(&mut &serialized.grandpa_authority_set.0[..])?,
finalized_block_header: codec::Decode::decode(
&mut &serialized.finalized_block_header.0[..],
)?,
babe_epoch_changes: codec::Decode::decode(&mut &serialized.babe_epoch_changes.0[..])?,
babe_finalized_block_weight: serialized.babe_finalized_block_weight,
grandpa_authority_set: codec::Decode::decode(
&mut &serialized.grandpa_authority_set.0[..],
)?,
})
}
}
@@ -469,12 +470,9 @@ mod tests {
struct Genesis(HashMap<String, String>);
impl BuildStorage for Genesis {
fn assimilate_storage(
&self,
storage: &mut Storage,
) -> Result<(), String> {
fn assimilate_storage(&self, storage: &mut Storage) -> Result<(), String> {
storage.top.extend(
self.0.iter().map(|(a, b)| (a.clone().into_bytes(), b.clone().into_bytes()))
self.0.iter().map(|(a, b)| (a.clone().into_bytes(), b.clone().into_bytes())),
);
Ok(())
}
@@ -485,11 +483,10 @@ mod tests {
#[test]
fn should_deserialize_example_chain_spec() {
let spec1 = TestSpec::from_json_bytes(Cow::Owned(
include_bytes!("../res/chain_spec.json").to_vec()
)).unwrap();
let spec2 = TestSpec::from_json_file(
PathBuf::from("./res/chain_spec.json")
).unwrap();
include_bytes!("../res/chain_spec.json").to_vec(),
))
.unwrap();
let spec2 = TestSpec::from_json_file(PathBuf::from("./res/chain_spec.json")).unwrap();
assert_eq!(spec1.as_json(false), spec2.as_json(false));
assert_eq!(spec2.chain_type(), ChainType::Live)
@@ -506,8 +503,9 @@ mod tests {
#[test]
fn should_deserialize_chain_spec_with_extensions() {
let spec = TestSpec2::from_json_bytes(Cow::Owned(
include_bytes!("../res/chain_spec2.json").to_vec()
)).unwrap();
include_bytes!("../res/chain_spec2.json").to_vec(),
))
.unwrap();
assert_eq!(spec.extensions().my_property, "Test Extension");
}
+70 -58
View File
@@ -18,19 +18,21 @@
//! Chain Spec extensions helpers.
use std::fmt::Debug;
use std::any::{TypeId, Any};
use std::{
any::{Any, TypeId},
fmt::Debug,
};
use std::collections::BTreeMap;
use serde::{Serialize, Deserialize, de::DeserializeOwned};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
/// A `ChainSpec` extension.
///
/// This trait is implemented automatically by `ChainSpecGroup` macro.
pub trait Group: Clone + Sized {
/// An associated type containing fork definition.
type Fork: Fork<Base=Self>;
type Fork: Fork<Base = Self>;
/// Convert to fork type.
fn to_fork(self) -> Self::Fork;
@@ -45,7 +47,7 @@ pub trait Group: Clone + Sized {
/// a complete set of parameters
pub trait Fork: Serialize + DeserializeOwned + Clone + Sized {
/// A base `Group` type.
type Base: Group<Fork=Self>;
type Base: Group<Fork = Self>;
/// Combine with another struct.
///
@@ -128,7 +130,8 @@ pub trait Extension: Serialize + DeserializeOwned + Clone {
fn get_any(&self, t: TypeId) -> &dyn Any;
/// Get forkable extensions of specific type.
fn forks<BlockNumber, T>(&self) -> Option<Forks<BlockNumber, T>> where
fn forks<BlockNumber, T>(&self) -> Option<Forks<BlockNumber, T>>
where
BlockNumber: Ord + Clone + 'static,
T: Group + 'static,
<Self::Forks as IsForks>::Extension: Extension,
@@ -142,8 +145,12 @@ pub trait Extension: Serialize + DeserializeOwned + Clone {
impl Extension for crate::NoExtension {
type Forks = Self;
fn get<T: 'static>(&self) -> Option<&T> { None }
fn get_any(&self, _t: TypeId) -> &dyn Any { self }
fn get<T: 'static>(&self) -> Option<&T> {
None
}
fn get_any(&self, _t: TypeId) -> &dyn Any {
self
}
}
pub trait IsForks {
@@ -166,14 +173,12 @@ pub struct Forks<BlockNumber: Ord, T: Group> {
impl<B: Ord, T: Group + Default> Default for Forks<B, T> {
fn default() -> Self {
Self {
base: Default::default(),
forks: Default::default(),
}
Self { base: Default::default(), forks: Default::default() }
}
}
impl<B: Ord, T: Group> Forks<B, T> where
impl<B: Ord, T: Group> Forks<B, T>
where
T::Fork: Debug,
{
/// Create new fork definition given the base and the forks.
@@ -195,7 +200,8 @@ impl<B: Ord, T: Group> Forks<B, T> where
}
}
impl<B, T> IsForks for Forks<B, T> where
impl<B, T> IsForks for Forks<B, T>
where
B: Ord + 'static,
T: Group + 'static,
{
@@ -203,29 +209,31 @@ impl<B, T> IsForks for Forks<B, T> where
type Extension = T;
}
impl<B: Ord + Clone, T: Group + Extension> Forks<B, T> where
impl<B: Ord + Clone, T: Group + Extension> Forks<B, T>
where
T::Fork: Extension,
{
/// Get forks definition for a subset of this extension.
///
/// Returns the `Forks` struct, but limited to a particular type
/// within the extension.
pub fn for_type<X>(&self) -> Option<Forks<B, X>> where
pub fn for_type<X>(&self) -> Option<Forks<B, X>>
where
X: Group + 'static,
{
let base = self.base.get::<X>()?.clone();
let forks = self.forks.iter().filter_map(|(k, v)| {
Some((k.clone(), v.get::<Option<X::Fork>>()?.clone()?))
}).collect();
let forks = self
.forks
.iter()
.filter_map(|(k, v)| Some((k.clone(), v.get::<Option<X::Fork>>()?.clone()?)))
.collect();
Some(Forks {
base,
forks,
})
Some(Forks { base, forks })
}
}
impl<B, E> Extension for Forks<B, E> where
impl<B, E> Extension for Forks<B, E>
where
B: Serialize + DeserializeOwned + Ord + Clone + 'static,
E: Extension + Group + 'static,
{
@@ -245,7 +253,8 @@ impl<B, E> Extension for Forks<B, E> where
}
}
fn forks<BlockNumber, T>(&self) -> Option<Forks<BlockNumber, T>> where
fn forks<BlockNumber, T>(&self) -> Option<Forks<BlockNumber, T>>
where
BlockNumber: Ord + Clone + 'static,
T: Group + 'static,
<Self::Forks as IsForks>::Extension: Extension,
@@ -266,7 +275,7 @@ pub trait GetExtension {
fn get_any(&self, t: TypeId) -> &dyn Any;
}
impl <E: Extension> GetExtension for E {
impl<E: Extension> GetExtension for E {
fn get_any(&self, t: TypeId) -> &dyn Any {
Extension::get_any(self, t)
}
@@ -281,7 +290,7 @@ pub fn get_extension<T: 'static>(e: &dyn GetExtension) -> Option<&T> {
#[cfg(test)]
mod tests {
use super::*;
use sc_chain_spec_derive::{ChainSpecGroup, ChainSpecExtension};
use sc_chain_spec_derive::{ChainSpecExtension, ChainSpecGroup};
// Make the proc macro work for tests and doc tests.
use crate as sc_chain_spec;
@@ -297,7 +306,9 @@ mod tests {
pub test: u8,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ChainSpecGroup, ChainSpecExtension)]
#[derive(
Debug, Clone, PartialEq, Serialize, Deserialize, ChainSpecGroup, ChainSpecExtension,
)]
#[serde(deny_unknown_fields)]
pub struct Extensions {
pub ext1: Extension1,
@@ -315,11 +326,12 @@ mod tests {
#[test]
fn forks_should_work_correctly() {
use super::Extension as _ ;
use super::Extension as _;
// We first need to deserialize into a `Value` because of the following bug:
// https://github.com/serde-rs/json/issues/505
let ext_val: serde_json::Value = serde_json::from_str(r#"
let ext_val: serde_json::Value = serde_json::from_str(
r#"
{
"test": 11,
"forkable": {
@@ -342,40 +354,40 @@ mod tests {
}
}
}
"#).unwrap();
"#,
)
.unwrap();
let ext: Ext2 = serde_json::from_value(ext_val).unwrap();
assert_eq!(ext.get::<Extension1>(), Some(&Extension1 {
test: 11
}));
assert_eq!(ext.get::<Extension1>(), Some(&Extension1 { test: 11 }));
// get forks definition
let forks = ext.get::<Forks<u64, Extensions>>().unwrap();
assert_eq!(forks.at_block(0), Extensions {
ext1: Extension1 { test: 15 },
ext2: Extension2 { test: 123 },
});
assert_eq!(forks.at_block(1), Extensions {
ext1: Extension1 { test: 5 },
ext2: Extension2 { test: 123 },
});
assert_eq!(forks.at_block(2), Extensions {
ext1: Extension1 { test: 5 },
ext2: Extension2 { test: 5 },
});
assert_eq!(forks.at_block(4), Extensions {
ext1: Extension1 { test: 5 },
ext2: Extension2 { test: 5 },
});
assert_eq!(forks.at_block(5), Extensions {
ext1: Extension1 { test: 5 },
ext2: Extension2 { test: 1 },
});
assert_eq!(forks.at_block(10), Extensions {
ext1: Extension1 { test: 5 },
ext2: Extension2 { test: 1 },
});
assert_eq!(
forks.at_block(0),
Extensions { ext1: Extension1 { test: 15 }, ext2: Extension2 { test: 123 } }
);
assert_eq!(
forks.at_block(1),
Extensions { ext1: Extension1 { test: 5 }, ext2: Extension2 { test: 123 } }
);
assert_eq!(
forks.at_block(2),
Extensions { ext1: Extension1 { test: 5 }, ext2: Extension2 { test: 5 } }
);
assert_eq!(
forks.at_block(4),
Extensions { ext1: Extension1 { test: 5 }, ext2: Extension2 { test: 5 } }
);
assert_eq!(
forks.at_block(5),
Extensions { ext1: Extension1 { test: 5 }, ext2: Extension2 { test: 1 } }
);
assert_eq!(
forks.at_block(10),
Extensions { ext1: Extension1 { test: 5 }, ext2: Extension2 { test: 1 } }
);
assert!(forks.at_block(10).get::<Extension2>().is_some());
// filter forks for `Extension2`
+16 -16
View File
@@ -35,7 +35,7 @@
//!
//! #[derive(Clone, Debug, serde::Serialize, serde::Deserialize, ChainSpecExtension)]
//! pub struct MyExtension {
//! pub known_blocks: HashMap<u64, String>,
//! pub known_blocks: HashMap<u64, String>,
//! }
//!
//! pub type MyChainSpec<G> = GenericChainSpec<G, MyExtension>;
@@ -53,19 +53,19 @@
//!
//! #[derive(Clone, Debug, serde::Serialize, serde::Deserialize, ChainSpecGroup)]
//! pub struct ClientParams {
//! max_block_size: usize,
//! max_extrinsic_size: usize,
//! max_block_size: usize,
//! max_extrinsic_size: usize,
//! }
//!
//! #[derive(Clone, Debug, serde::Serialize, serde::Deserialize, ChainSpecGroup)]
//! pub struct PoolParams {
//! max_transaction_size: usize,
//! max_transaction_size: usize,
//! }
//!
//! #[derive(Clone, Debug, serde::Serialize, serde::Deserialize, ChainSpecGroup, ChainSpecExtension)]
//! pub struct Extension {
//! pub client: ClientParams,
//! pub pool: PoolParams,
//! pub client: ClientParams,
//! pub pool: PoolParams,
//! }
//!
//! pub type BlockNumber = u64;
@@ -88,20 +88,20 @@
//!
//! #[derive(Clone, Debug, Serialize, Deserialize, ChainSpecGroup)]
//! pub struct ClientParams {
//! max_block_size: usize,
//! max_extrinsic_size: usize,
//! max_block_size: usize,
//! max_extrinsic_size: usize,
//! }
//!
//! #[derive(Clone, Debug, Serialize, Deserialize, ChainSpecGroup)]
//! pub struct PoolParams {
//! max_transaction_size: usize,
//! max_transaction_size: usize,
//! }
//!
//! #[derive(Clone, Debug, Serialize, Deserialize, ChainSpecExtension)]
//! pub struct Extension {
//! pub client: ClientParams,
//! #[forks]
//! pub pool: Forks<u64, PoolParams>,
//! pub client: ClientParams,
//! #[forks]
//! pub pool: Forks<u64, PoolParams>,
//! }
//!
//! pub type MyChainSpec<G> = GenericChainSpec<G, Extension>;
@@ -111,16 +111,16 @@ mod chain_spec;
mod extension;
pub use chain_spec::{
ChainSpec as GenericChainSpec, NoExtension, LightSyncState, SerializableLightSyncState,
ChainSpec as GenericChainSpec, LightSyncState, NoExtension, SerializableLightSyncState,
};
pub use extension::{Group, Fork, Forks, Extension, GetExtension, get_extension};
pub use extension::{get_extension, Extension, Fork, Forks, GetExtension, Group};
pub use sc_chain_spec_derive::{ChainSpecExtension, ChainSpecGroup};
use serde::{Serialize, de::DeserializeOwned};
use sp_runtime::BuildStorage;
use sc_network::config::MultiaddrWithPeerId;
use sc_telemetry::TelemetryEndpoints;
use serde::{de::DeserializeOwned, Serialize};
use sp_core::storage::Storage;
use sp_runtime::BuildStorage;
/// The type of a chain.
///
+6 -11
View File
@@ -74,9 +74,8 @@ impl WasmExecutionMethod {
impl Into<sc_service::config::WasmExecutionMethod> for WasmExecutionMethod {
fn into(self) -> sc_service::config::WasmExecutionMethod {
match self {
WasmExecutionMethod::Interpreted => {
sc_service::config::WasmExecutionMethod::Interpreted
}
WasmExecutionMethod::Interpreted =>
sc_service::config::WasmExecutionMethod::Interpreted,
#[cfg(feature = "wasmtime")]
WasmExecutionMethod::Compiled => sc_service::config::WasmExecutionMethod::Compiled,
#[cfg(not(feature = "wasmtime"))]
@@ -250,14 +249,10 @@ impl Into<sc_network::config::SyncMode> for SyncMode {
fn into(self) -> sc_network::config::SyncMode {
match self {
SyncMode::Full => sc_network::config::SyncMode::Full,
SyncMode::Fast => sc_network::config::SyncMode::Fast {
skip_proofs: false,
storage_chain_mode: false,
},
SyncMode::FastUnsafe => sc_network::config::SyncMode::Fast {
skip_proofs: true,
storage_chain_mode: false,
},
SyncMode::Fast =>
sc_network::config::SyncMode::Fast { skip_proofs: false, storage_chain_mode: false },
SyncMode::FastUnsafe =>
sc_network::config::SyncMode::Fast { skip_proofs: true, storage_chain_mode: false },
}
}
}
@@ -16,15 +16,19 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::error;
use crate::params::NodeKeyParams;
use crate::params::SharedParams;
use crate::CliConfiguration;
use crate::{
error,
params::{NodeKeyParams, SharedParams},
CliConfiguration,
};
use log::info;
use sc_network::config::build_multiaddr;
use sc_service::{config::{MultiaddrWithPeerId, NetworkConfiguration}, ChainSpec};
use structopt::StructOpt;
use sc_service::{
config::{MultiaddrWithPeerId, NetworkConfiguration},
ChainSpec,
};
use std::io::Write;
use structopt::StructOpt;
/// The `build-spec` command used to build a specification.
#[derive(Debug, StructOpt, Clone)]
@@ -17,7 +17,9 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
CliConfiguration, error, params::{ImportParams, SharedParams, BlockNumberOrHash},
error,
params::{BlockNumberOrHash, ImportParams, SharedParams},
CliConfiguration,
};
use sc_client_api::{BlockBackend, UsageProvider};
use sp_runtime::traits::{Block as BlockT, Header as HeaderT};
@@ -48,11 +50,7 @@ pub struct CheckBlockCmd {
impl CheckBlockCmd {
/// Run the check-block command
pub async fn run<B, C, IQ>(
&self,
client: Arc<C>,
import_queue: IQ,
) -> error::Result<()>
pub async fn run<B, C, IQ>(&self, client: Arc<C>, import_queue: IQ) -> error::Result<()>
where
B: BlockT + for<'de> serde::Deserialize<'de>,
C: BlockBackend<B> + UsageProvider<B> + Send + Sync + 'static,
@@ -16,21 +16,16 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::error;
use crate::params::{GenericNumber, DatabaseParams, PruningParams, SharedParams};
use crate::CliConfiguration;
use log::info;
use sc_service::{
config::DatabaseConfig, chain_ops::export_blocks,
use crate::{
error,
params::{DatabaseParams, GenericNumber, PruningParams, SharedParams},
CliConfiguration,
};
use log::info;
use sc_client_api::{BlockBackend, UsageProvider};
use sc_service::{chain_ops::export_blocks, config::DatabaseConfig};
use sp_runtime::traits::{Block as BlockT, Header as HeaderT};
use std::fmt::Debug;
use std::fs;
use std::io;
use std::path::PathBuf;
use std::str::FromStr;
use std::sync::Arc;
use std::{fmt::Debug, fs, io, path::PathBuf, str::FromStr, sync::Arc};
use structopt::StructOpt;
/// The `export-blocks` command used to export blocks.
@@ -95,9 +90,7 @@ impl ExportBlocksCmd {
None => Box::new(io::stdout()),
};
export_blocks(client, file, from.into(), to, binary)
.await
.map_err(Into::into)
export_blocks(client, file, from.into(), to, binary).await.map_err(Into::into)
}
}
@@ -17,13 +17,15 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
CliConfiguration, error, params::{PruningParams, SharedParams, BlockNumberOrHash},
error,
params::{BlockNumberOrHash, PruningParams, SharedParams},
CliConfiguration,
};
use log::info;
use sp_runtime::traits::{Block as BlockT, Header as HeaderT};
use std::{fmt::Debug, str::FromStr, io::Write, sync::Arc};
use structopt::StructOpt;
use sc_client_api::{StorageProvider, UsageProvider};
use sp_runtime::traits::{Block as BlockT, Header as HeaderT};
use std::{fmt::Debug, io::Write, str::FromStr, sync::Arc};
use structopt::StructOpt;
/// The `export-state` command used to export the state of a given block into
/// a chain spec.
+9 -10
View File
@@ -16,12 +16,12 @@
// limitations under the License.
//! Implementation of the `generate` subcommand
use bip39::{MnemonicType, Mnemonic, Language};
use structopt::StructOpt;
use crate::{
utils::print_from_uri, KeystoreParams, Error,
with_crypto_scheme, NetworkSchemeFlag, OutputTypeFlag, CryptoSchemeFlag,
utils::print_from_uri, with_crypto_scheme, CryptoSchemeFlag, Error, KeystoreParams,
NetworkSchemeFlag, OutputTypeFlag,
};
use bip39::{Language, Mnemonic, MnemonicType};
use structopt::StructOpt;
/// The `generate` command
#[derive(Debug, StructOpt, Clone)]
@@ -52,12 +52,11 @@ impl GenerateCmd {
/// Run the command
pub fn run(&self) -> Result<(), Error> {
let words = match self.words {
Some(words) => {
MnemonicType::for_word_count(words)
.map_err(|_| {
Error::Input("Invalid number of words given for phrase: must be 12/15/18/21/24".into())
})?
},
Some(words) => MnemonicType::for_word_count(words).map_err(|_| {
Error::Input(
"Invalid number of words given for phrase: must be 12/15/18/21/24".into(),
)
})?,
None => MnemonicType::Words12,
};
let mnemonic = Mnemonic::new(words, Language::English);
@@ -18,9 +18,9 @@
//! Implementation of the `generate-node-key` subcommand
use crate::Error;
use structopt::StructOpt;
use std::{path::PathBuf, fs};
use libp2p::identity::{ed25519 as libp2p_ed25519, PublicKey};
use std::{fs, path::PathBuf};
use structopt::StructOpt;
/// The `generate-node-key` command
#[derive(Debug, StructOpt)]
@@ -59,15 +59,14 @@ impl GenerateNodeKeyCmd {
#[cfg(test)]
mod tests {
use super::*;
use tempfile::Builder;
use std::io::Read;
use tempfile::Builder;
#[test]
fn generate_node_key() {
let mut file = Builder::new().prefix("keyfile").tempfile().unwrap();
let file_path = file.path().display().to_string();
let generate =
GenerateNodeKeyCmd::from_iter(&["generate-node-key", "--file", &file_path]);
let generate = GenerateNodeKeyCmd::from_iter(&["generate-node-key", "--file", &file_path]);
assert!(generate.run().is_ok());
let mut buf = String::new();
assert!(file.read_to_string(&mut buf).is_ok());
@@ -16,19 +16,22 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::error;
use crate::params::ImportParams;
use crate::params::SharedParams;
use crate::CliConfiguration;
use crate::{
error,
params::{ImportParams, SharedParams},
CliConfiguration,
};
use sc_client_api::UsageProvider;
use sc_service::chain_ops::import_blocks;
use sp_runtime::traits::Block as BlockT;
use std::fmt::Debug;
use std::fs;
use std::io::{self, Read, Seek};
use std::path::PathBuf;
use std::sync::Arc;
use std::{
fmt::Debug,
fs,
io::{self, Read, Seek},
path::PathBuf,
sync::Arc,
};
use structopt::StructOpt;
use sc_client_api::UsageProvider;
/// The `import-blocks` command used to import blocks.
#[derive(Debug, StructOpt)]
@@ -63,11 +66,7 @@ impl<T: Read + Seek> ReadPlusSeek for T {}
impl ImportBlocksCmd {
/// Run the import-blocks command
pub async fn run<B, C, IQ>(
&self,
client: Arc<C>,
import_queue: IQ,
) -> error::Result<()>
pub async fn run<B, C, IQ>(&self, client: Arc<C>, import_queue: IQ) -> error::Result<()>
where
C: UsageProvider<B> + Send + Sync + 'static,
B: BlockT + for<'de> serde::Deserialize<'de>,
@@ -79,7 +78,7 @@ impl ImportBlocksCmd {
let mut buffer = Vec::new();
io::stdin().read_to_end(&mut buffer)?;
Box::new(io::Cursor::new(buffer))
}
},
};
import_blocks(client, import_queue, file, false, self.binary)
+37 -38
View File
@@ -18,22 +18,18 @@
//! Implementation of the `insert` subcommand
use crate::{
Error, KeystoreParams, CryptoSchemeFlag, SharedParams, utils, with_crypto_scheme,
SubstrateCli,
utils, with_crypto_scheme, CryptoSchemeFlag, Error, KeystoreParams, SharedParams, SubstrateCli,
};
use std::{sync::Arc, convert::TryFrom};
use structopt::StructOpt;
use sp_core::{crypto::KeyTypeId, crypto::SecretString};
use sp_keystore::{SyncCryptoStorePtr, SyncCryptoStore};
use sc_keystore::LocalKeystore;
use sc_service::config::{KeystoreConfig, BasePath};
use sc_service::config::{BasePath, KeystoreConfig};
use sp_core::crypto::{KeyTypeId, SecretString};
use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr};
use std::{convert::TryFrom, sync::Arc};
use structopt::StructOpt;
/// The `insert` command
#[derive(Debug, StructOpt, Clone)]
#[structopt(
name = "insert",
about = "Insert a key to the keystore of a node."
)]
#[structopt(name = "insert", about = "Insert a key to the keystore of a node.")]
pub struct InsertKeyCmd {
/// The secret key URI.
/// If the value is a file, the file content is used as URI.
@@ -62,7 +58,8 @@ impl InsertKeyCmd {
/// Run the command
pub fn run<C: SubstrateCli>(&self, cli: &C) -> Result<(), Error> {
let suri = utils::read_uri(self.suri.as_ref())?;
let base_path = self.shared_params
let base_path = self
.shared_params
.base_path()
.unwrap_or_else(|| BasePath::from_project("", "", &C::executable_name()));
let chain_id = self.shared_params.chain_id(self.shared_params.is_dev());
@@ -78,10 +75,11 @@ impl InsertKeyCmd {
let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::open(path, password)?);
(keystore, public)
},
_ => unreachable!("keystore_config always returns path and password; qed")
_ => unreachable!("keystore_config always returns path and password; qed"),
};
let key_type = KeyTypeId::try_from(self.key_type.as_str()).map_err(|_| Error::KeyTypeInvalid)?;
let key_type =
KeyTypeId::try_from(self.key_type.as_str()).map_err(|_| Error::KeyTypeInvalid)?;
SyncCryptoStore::insert_unknown(&*keystore, key_type, &suri, &public[..])
.map_err(|_| Error::KeyStoreOperation)?;
@@ -98,10 +96,10 @@ fn to_vec<P: sp_core::Pair>(uri: &str, pass: Option<SecretString>) -> Result<Vec
#[cfg(test)]
mod tests {
use super::*;
use sc_service::{ChainSpec, ChainType, GenericChainSpec, NoExtension};
use sp_core::{sr25519::Pair, Pair as _, Public};
use structopt::StructOpt;
use tempfile::TempDir;
use sp_core::{sr25519::Pair, Pair as _, Public};
use sc_service::{ChainSpec, GenericChainSpec, ChainType, NoExtension};
struct Cli;
@@ -135,21 +133,17 @@ mod tests {
}
fn load_spec(&self, _: &str) -> std::result::Result<Box<dyn ChainSpec>, String> {
Ok(
Box::new(
GenericChainSpec::from_genesis(
"test",
"test_id",
ChainType::Development,
|| unimplemented!("Not required in tests"),
Vec::new(),
None,
None,
None,
NoExtension::None,
),
),
)
Ok(Box::new(GenericChainSpec::from_genesis(
"test",
"test_id",
ChainType::Development,
|| unimplemented!("Not required in tests"),
Vec::new(),
None,
None,
None,
NoExtension::None,
)))
}
}
@@ -159,15 +153,20 @@ mod tests {
let path_str = format!("{}", path.path().display());
let (key, uri, _) = Pair::generate_with_phrase(None);
let inspect = InsertKeyCmd::from_iter(
&["insert-key", "-d", &path_str, "--key-type", "test", "--suri", &uri],
);
let inspect = InsertKeyCmd::from_iter(&[
"insert-key",
"-d",
&path_str,
"--key-type",
"test",
"--suri",
&uri,
]);
assert!(inspect.run(&Cli).is_ok());
let keystore = LocalKeystore::open(
path.path().join("chains").join("test_id").join("keystore"),
None,
).unwrap();
let keystore =
LocalKeystore::open(path.path().join("chains").join("test_id").join("keystore"), None)
.unwrap();
assert!(keystore.has_keys(&[(key.public().to_raw_vec(), KeyTypeId(*b"test"))]));
}
}
@@ -18,8 +18,8 @@
//! Implementation of the `inspect` subcommand
use crate::{
utils::{self, print_from_uri, print_from_public}, KeystoreParams,
with_crypto_scheme, NetworkSchemeFlag, OutputTypeFlag, CryptoSchemeFlag, Error,
utils::{self, print_from_public, print_from_uri},
with_crypto_scheme, CryptoSchemeFlag, Error, KeystoreParams, NetworkSchemeFlag, OutputTypeFlag,
};
use structopt::StructOpt;
/// The `inspect` command
@@ -103,8 +103,7 @@ mod tests {
"remember fiber forum demise paper uniform squirrel feel access exclude casual effort";
let seed = "0xad1fb77243b536b90cfe5f0d351ab1b1ac40e3890b41dc64f766ee56340cfca5";
let inspect =
InspectKeyCmd::from_iter(&["inspect-key", words, "--password", "12345"]);
let inspect = InspectKeyCmd::from_iter(&["inspect-key", words, "--password", "12345"]);
assert!(inspect.run().is_ok());
let inspect = InspectKeyCmd::from_iter(&["inspect-key", seed]);
@@ -18,9 +18,8 @@
//! Implementation of the `inspect-node-key` subcommand
use crate::{Error, NetworkSchemeFlag};
use std::fs;
use libp2p::identity::{PublicKey, ed25519};
use std::path::PathBuf;
use libp2p::identity::{ed25519, PublicKey};
use std::{fs, path::PathBuf};
use structopt::StructOpt;
/// The `inspect-node-key` command
@@ -42,10 +41,10 @@ pub struct InspectNodeKeyCmd {
impl InspectNodeKeyCmd {
/// runs the command
pub fn run(&self) -> Result<(), Error> {
let mut file_content = hex::decode(fs::read(&self.file)?)
.map_err(|_| "failed to decode secret as hex")?;
let secret = ed25519::SecretKey::from_bytes(&mut file_content)
.map_err(|_| "Bad node key file")?;
let mut file_content =
hex::decode(fs::read(&self.file)?).map_err(|_| "failed to decode secret as hex")?;
let secret =
ed25519::SecretKey::from_bytes(&mut file_content).map_err(|_| "Bad node key file")?;
let keypair = ed25519::Keypair::from(secret);
let peer_id = PublicKey::Ed25519(keypair.public()).into_peer_id();
@@ -58,8 +57,7 @@ impl InspectNodeKeyCmd {
#[cfg(test)]
mod tests {
use super::*;
use super::super::GenerateNodeKeyCmd;
use super::{super::GenerateNodeKeyCmd, *};
#[test]
fn inspect_node_key() {
+2 -5
View File
@@ -21,11 +21,8 @@ use crate::{Error, SubstrateCli};
use structopt::StructOpt;
use super::{
insert_key::InsertKeyCmd,
inspect_key::InspectKeyCmd,
generate::GenerateCmd,
inspect_node_key::InspectNodeKeyCmd,
generate_node_key::GenerateNodeKeyCmd,
generate::GenerateCmd, generate_node_key::GenerateNodeKeyCmd, insert_key::InsertKeyCmd,
inspect_key::InspectKeyCmd, inspect_node_key::InspectNodeKeyCmd,
};
/// Key utilities for the cli.
+15 -26
View File
@@ -19,37 +19,26 @@ mod build_spec_cmd;
mod check_block_cmd;
mod export_blocks_cmd;
mod export_state_cmd;
mod generate;
mod generate_node_key;
mod import_blocks_cmd;
mod insert_key;
mod inspect_key;
mod inspect_node_key;
mod key;
mod purge_chain_cmd;
mod sign;
mod verify;
mod vanity;
mod revert_cmd;
mod run_cmd;
mod generate_node_key;
mod generate;
mod insert_key;
mod inspect_node_key;
mod inspect_key;
mod key;
mod sign;
pub mod utils;
mod vanity;
mod verify;
pub use self::{
build_spec_cmd::BuildSpecCmd,
check_block_cmd::CheckBlockCmd,
export_blocks_cmd::ExportBlocksCmd,
export_state_cmd::ExportStateCmd,
import_blocks_cmd::ImportBlocksCmd,
purge_chain_cmd::PurgeChainCmd,
sign::SignCmd,
generate::GenerateCmd,
insert_key::InsertKeyCmd,
inspect_key::InspectKeyCmd,
generate_node_key::GenerateNodeKeyCmd,
inspect_node_key::InspectNodeKeyCmd,
key::KeySubcommand,
vanity::VanityCmd,
verify::VerifyCmd,
revert_cmd::RevertCmd,
run_cmd::RunCmd,
build_spec_cmd::BuildSpecCmd, check_block_cmd::CheckBlockCmd,
export_blocks_cmd::ExportBlocksCmd, export_state_cmd::ExportStateCmd, generate::GenerateCmd,
generate_node_key::GenerateNodeKeyCmd, import_blocks_cmd::ImportBlocksCmd,
insert_key::InsertKeyCmd, inspect_key::InspectKeyCmd, inspect_node_key::InspectNodeKeyCmd,
key::KeySubcommand, purge_chain_cmd::PurgeChainCmd, revert_cmd::RevertCmd, run_cmd::RunCmd,
sign::SignCmd, vanity::VanityCmd, verify::VerifyCmd,
};
@@ -16,13 +16,17 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::error;
use crate::params::{DatabaseParams, SharedParams};
use crate::CliConfiguration;
use crate::{
error,
params::{DatabaseParams, SharedParams},
CliConfiguration,
};
use sc_service::DatabaseConfig;
use std::fmt::Debug;
use std::fs;
use std::io::{self, Write};
use std::{
fmt::Debug,
fs,
io::{self, Write},
};
use structopt::StructOpt;
/// The `purge-chain` command used to remove the whole chain.
@@ -44,10 +48,9 @@ pub struct PurgeChainCmd {
impl PurgeChainCmd {
/// Run the purge command
pub fn run(&self, database_config: DatabaseConfig) -> error::Result<()> {
let db_path = database_config.path()
.ok_or_else(||
error::Error::Input("Cannot purge custom database implementation".into())
)?;
let db_path = database_config.path().ok_or_else(|| {
error::Error::Input("Cannot purge custom database implementation".into())
})?;
if !self.yes {
print!("Are you sure to remove {:?}? [y/N]: ", &db_path);
@@ -61,7 +64,7 @@ impl PurgeChainCmd {
Some('y') | Some('Y') => {},
_ => {
println!("Aborted");
return Ok(());
return Ok(())
},
}
}
@@ -16,16 +16,16 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::error;
use crate::params::{GenericNumber, PruningParams, SharedParams};
use crate::CliConfiguration;
use crate::{
error,
params::{GenericNumber, PruningParams, SharedParams},
CliConfiguration,
};
use sc_client_api::{Backend, UsageProvider};
use sc_service::chain_ops::revert_chain;
use sp_runtime::traits::{Block as BlockT, Header as HeaderT};
use std::fmt::Debug;
use std::str::FromStr;
use std::sync::Arc;
use std::{fmt::Debug, str::FromStr, sync::Arc};
use structopt::StructOpt;
use sc_client_api::{Backend, UsageProvider};
/// The `revert` command used revert the chain to a previous state.
#[derive(Debug, StructOpt)]
@@ -45,11 +45,7 @@ pub struct RevertCmd {
impl RevertCmd {
/// Run the revert command
pub async fn run<B, BA, C>(
&self,
client: Arc<C>,
backend: Arc<BA>,
) -> error::Result<()>
pub async fn run<B, BA, C>(&self, client: Arc<C>, backend: Arc<BA>) -> error::Result<()>
where
B: BlockT,
BA: Backend<B>,
+29 -38
View File
@@ -16,15 +16,15 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::arg_enums::RpcMethods;
use crate::error::{Error, Result};
use crate::params::ImportParams;
use crate::params::KeystoreParams;
use crate::params::NetworkParams;
use crate::params::OffchainWorkerParams;
use crate::params::SharedParams;
use crate::params::TransactionPoolParams;
use crate::CliConfiguration;
use crate::{
arg_enums::RpcMethods,
error::{Error, Result},
params::{
ImportParams, KeystoreParams, NetworkParams, OffchainWorkerParams, SharedParams,
TransactionPoolParams,
},
CliConfiguration,
};
use regex::Regex;
use sc_service::{
config::{BasePath, PrometheusConfig, TransactionPoolOptions},
@@ -308,7 +308,7 @@ impl CliConfiguration for RunCmd {
Error::Input(format!(
"Invalid node name '{}'. Reason: {}. If unsure, use none.",
name, msg
))
))
})?;
Ok(name)
@@ -363,18 +363,13 @@ impl CliConfiguration for RunCmd {
Ok(if self.no_prometheus {
None
} else {
let interface = if self.prometheus_external {
Ipv4Addr::UNSPECIFIED
} else {
Ipv4Addr::LOCALHOST
};
let interface =
if self.prometheus_external { Ipv4Addr::UNSPECIFIED } else { Ipv4Addr::LOCALHOST };
Some(PrometheusConfig::new_with_default_registry(
SocketAddr::new(
interface.into(),
self.prometheus_port.unwrap_or(default_listen_port),
)
))
Some(PrometheusConfig::new_with_default_registry(SocketAddr::new(
interface.into(),
self.prometheus_port.unwrap_or(default_listen_port),
)))
})
}
@@ -416,7 +411,7 @@ impl CliConfiguration for RunCmd {
self.rpc_external,
self.unsafe_rpc_external,
self.rpc_methods,
self.validator
self.validator,
)?;
Ok(Some(SocketAddr::new(interface, self.rpc_port.unwrap_or(default_listen_port))))
@@ -466,19 +461,19 @@ impl CliConfiguration for RunCmd {
pub fn is_node_name_valid(_name: &str) -> std::result::Result<(), &str> {
let name = _name.to_string();
if name.chars().count() >= crate::NODE_NAME_MAX_LENGTH {
return Err("Node name too long");
return Err("Node name too long")
}
let invalid_chars = r"[\\.@]";
let re = Regex::new(invalid_chars).unwrap();
if re.is_match(&name) {
return Err("Node name should not contain invalid chars such as '.' and '@'");
return Err("Node name should not contain invalid chars such as '.' and '@'")
}
let invalid_patterns = r"(https?:\\/+)?(www)+";
let re = Regex::new(invalid_patterns).unwrap();
if re.is_match(&name) {
return Err("Node name should not contain urls");
return Err("Node name should not contain urls")
}
Ok(())
@@ -497,7 +492,7 @@ fn rpc_interface(
or `--rpc-methods=unsafe` if you understand the risks. See the options \
description for more information."
.to_owned(),
));
))
}
if is_external || is_unsafe_external {
@@ -537,11 +532,10 @@ fn parse_telemetry_endpoints(s: &str) -> std::result::Result<(String, u8), Telem
None => Err(TelemetryParsingError::MissingVerbosity),
Some(pos_) => {
let url = s[..pos_].to_string();
let verbosity = s[pos_ + 1..]
.parse()
.map_err(TelemetryParsingError::VerbosityParsingError)?;
let verbosity =
s[pos_ + 1..].parse().map_err(TelemetryParsingError::VerbosityParsingError)?;
Ok((url, verbosity))
}
},
}
}
@@ -574,17 +568,13 @@ fn parse_cors(s: &str) -> std::result::Result<Cors, Box<dyn std::error::Error>>
match part {
"all" | "*" => {
is_all = true;
break;
}
break
},
other => origins.push(other.to_owned()),
}
}
Ok(if is_all {
Cors::All
} else {
Cors::List(origins)
})
Ok(if is_all { Cors::All } else { Cors::List(origins) })
}
#[cfg(test)]
@@ -600,7 +590,8 @@ mod tests {
fn tests_node_name_bad() {
assert!(is_node_name_valid(
"very very long names are really not very cool for the ui at all, really they're not"
).is_err());
)
.is_err());
assert!(is_node_name_valid("Dots.not.Ok").is_err());
assert!(is_node_name_valid("http://visit.me").is_err());
assert!(is_node_name_valid("https://visit.me").is_err());
+11 -13
View File
@@ -5,7 +5,7 @@
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
@@ -18,15 +18,12 @@
//! Implementation of the `sign` subcommand
use crate::{error, utils, with_crypto_scheme, CryptoSchemeFlag, KeystoreParams};
use structopt::StructOpt;
use sp_core::crypto::SecretString;
use structopt::StructOpt;
/// The `sign` command
#[derive(Debug, StructOpt, Clone)]
#[structopt(
name = "sign",
about = "Sign a message, with a given (secret) key"
)]
#[structopt(name = "sign", about = "Sign a message, with a given (secret) key")]
pub struct SignCmd {
/// The secret key URI.
/// If the value is a file, the file content is used as URI.
@@ -52,7 +49,6 @@ pub struct SignCmd {
pub crypto_scheme: CryptoSchemeFlag,
}
impl SignCmd {
/// Run the command
pub fn run(&self) -> error::Result<()> {
@@ -60,17 +56,19 @@ impl SignCmd {
let suri = utils::read_uri(self.suri.as_ref())?;
let password = self.keystore_params.read_password()?;
let signature = with_crypto_scheme!(
self.crypto_scheme.scheme,
sign(&suri, password, message)
)?;
let signature =
with_crypto_scheme!(self.crypto_scheme.scheme, sign(&suri, password, message))?;
println!("{}", signature);
Ok(())
}
}
fn sign<P: sp_core::Pair>(suri: &str, password: Option<SecretString>, message: Vec<u8>) -> error::Result<String> {
fn sign<P: sp_core::Pair>(
suri: &str,
password: Option<SecretString>,
message: Vec<u8>,
) -> error::Result<String> {
let pair = utils::pair_from_suri::<P>(suri, password)?;
Ok(format!("{}", hex::encode(pair.sign(&message))))
}
@@ -91,7 +89,7 @@ mod test {
"--message",
&seed[2..],
"--password",
"12345"
"12345",
]);
assert!(sign.run().is_ok());
}
+16 -33
View File
@@ -22,9 +22,8 @@ use crate::{
OutputType,
};
use serde_json::json;
use sp_core::crypto::{ExposeSecret, SecretString, Zeroize};
use sp_core::{
crypto::{Ss58AddressFormat, Ss58Codec},
crypto::{ExposeSecret, SecretString, Ss58AddressFormat, Ss58Codec, Zeroize},
hexdisplay::HexDisplay,
Pair,
};
@@ -88,7 +87,7 @@ pub fn print_from_uri<Pair>(
"{}",
serde_json::to_string_pretty(&json).expect("Json pretty print failed")
);
}
},
OutputType::Text => {
println!(
"Secret phrase `{}` is account:\n \
@@ -102,12 +101,9 @@ pub fn print_from_uri<Pair>(
format_public_key::<Pair>(public_key.clone()),
public_key.to_ss58check_with_version(network_override),
format_account_id::<Pair>(public_key),
pair.public()
.into()
.into_account()
.to_ss58check_with_version(network_override),
pair.public().into().into_account().to_ss58check_with_version(network_override),
);
}
},
}
} else if let Ok((pair, seed)) = Pair::from_string_with_seed(uri, password.clone()) {
let public_key = pair.public();
@@ -127,7 +123,7 @@ pub fn print_from_uri<Pair>(
"{}",
serde_json::to_string_pretty(&json).expect("Json pretty print failed")
);
}
},
OutputType::Text => {
println!(
"Secret Key URI `{}` is account:\n \
@@ -137,20 +133,13 @@ pub fn print_from_uri<Pair>(
Account ID: {}\n \
SS58 Address: {}",
uri,
if let Some(seed) = seed {
format_seed::<Pair>(seed)
} else {
"n/a".into()
},
if let Some(seed) = seed { format_seed::<Pair>(seed) } else { "n/a".into() },
format_public_key::<Pair>(public_key.clone()),
public_key.to_ss58check_with_version(network_override),
format_account_id::<Pair>(public_key),
pair.public()
.into()
.into_account()
.to_ss58check_with_version(network_override),
pair.public().into().into_account().to_ss58check_with_version(network_override),
);
}
},
}
} else if let Ok((public_key, network)) = Pair::Public::from_string_with_version(uri) {
let network_override = network_override.unwrap_or(network);
@@ -170,7 +159,7 @@ pub fn print_from_uri<Pair>(
"{}",
serde_json::to_string_pretty(&json).expect("Json pretty print failed")
);
}
},
OutputType::Text => {
println!(
"Public Key URI `{}` is account:\n \
@@ -186,7 +175,7 @@ pub fn print_from_uri<Pair>(
format_account_id::<Pair>(public_key.clone()),
public_key.to_ss58check_with_version(network_override),
);
}
},
}
} else {
println!("Invalid phrase/URI given");
@@ -220,11 +209,8 @@ where
"ss58Address": public_key.to_ss58check_with_version(network_override),
});
println!(
"{}",
serde_json::to_string_pretty(&json).expect("Json pretty print failed")
);
}
println!("{}", serde_json::to_string_pretty(&json).expect("Json pretty print failed"));
},
OutputType::Text => {
println!(
"Network ID/version: {}\n \
@@ -238,7 +224,7 @@ where
format_account_id::<Pair>(public_key.clone()),
public_key.to_ss58check_with_version(network_override),
);
}
},
}
Ok(())
@@ -273,10 +259,7 @@ fn format_account_id<P: sp_core::Pair>(public_key: PublicFor<P>) -> String
where
PublicFor<P>: Into<MultiSigner>,
{
format!(
"0x{}",
HexDisplay::from(&public_key.into().into_account().as_ref())
)
format!("0x{}", HexDisplay::from(&public_key.into().into_account().as_ref()))
}
/// helper method for decoding hex
@@ -294,13 +277,13 @@ pub fn read_message(msg: Option<&String>, should_decode: bool) -> Result<Vec<u8>
match msg {
Some(m) => {
message = decode_hex(m)?;
}
},
None => {
std::io::stdin().lock().read_to_end(&mut message)?;
if should_decode {
message = decode_hex(&message)?;
}
}
},
}
Ok(message)
}
+29 -48
View File
@@ -5,7 +5,7 @@
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
@@ -19,21 +19,17 @@
//! implementation of the `vanity` subcommand
use crate::{
error, utils, with_crypto_scheme,
CryptoSchemeFlag, NetworkSchemeFlag, OutputTypeFlag,
error, utils, with_crypto_scheme, CryptoSchemeFlag, NetworkSchemeFlag, OutputTypeFlag,
};
use sp_core::crypto::{Ss58Codec, Ss58AddressFormat};
use structopt::StructOpt;
use rand::{rngs::OsRng, RngCore};
use sp_core::crypto::{Ss58AddressFormat, Ss58Codec};
use sp_runtime::traits::IdentifyAccount;
use structopt::StructOpt;
use utils::print_from_uri;
/// The `vanity` command
#[derive(Debug, StructOpt, Clone)]
#[structopt(
name = "vanity",
about = "Generate a seed that provides a vanity address"
)]
#[structopt(name = "vanity", about = "Generate a seed that provides a vanity address")]
pub struct VanityCmd {
/// Desired pattern
#[structopt(long, parse(try_from_str = assert_non_empty_string))]
@@ -78,10 +74,10 @@ fn generate_key<Pair>(
desired: &str,
network_override: Ss58AddressFormat,
) -> Result<String, &'static str>
where
Pair: sp_core::Pair,
Pair::Public: IdentifyAccount,
<Pair::Public as IdentifyAccount>::AccountId: Ss58Codec,
where
Pair: sp_core::Pair,
Pair::Public: IdentifyAccount,
<Pair::Public as IdentifyAccount>::AccountId: Ss58Codec,
{
println!("Generating key containing pattern '{}'", desired);
@@ -104,7 +100,7 @@ fn generate_key<Pair>(
best = score;
if best >= top {
println!("best: {} == top: {}", best, top);
return Ok(utils::format_seed::<Pair>(seed.clone()));
return Ok(utils::format_seed::<Pair>(seed.clone()))
}
}
done += 1;
@@ -129,11 +125,11 @@ fn next_seed(seed: &mut [u8]) {
match seed[i] {
255 => {
seed[i] = 0;
}
},
_ => {
seed[i] += 1;
break;
}
break
},
}
}
}
@@ -145,7 +141,7 @@ fn calculate_score(_desired: &str, key: &str) -> usize {
let snip_size = _desired.len() - truncate;
let truncated = &_desired[0..snip_size];
if let Some(pos) = key.find(truncated) {
return (47 - pos) + (snip_size * 48);
return (47 - pos) + (snip_size * 48)
}
}
0
@@ -160,15 +156,13 @@ fn assert_non_empty_string(pattern: &str) -> Result<String, &'static str> {
}
}
#[cfg(test)]
mod tests {
use super::*;
use sp_core::{crypto::Ss58Codec, Pair};
use sp_core::sr25519;
use sp_core::{crypto::Ss58Codec, sr25519, Pair};
use structopt::StructOpt;
#[cfg(feature = "bench")]
use test::Bencher;
use structopt::StructOpt;
#[test]
fn vanity() {
@@ -179,25 +173,21 @@ mod tests {
#[test]
fn test_generation_with_single_char() {
let seed = generate_key::<sr25519::Pair>("ab", Default::default()).unwrap();
assert!(
sr25519::Pair::from_seed_slice(&hex::decode(&seed[2..]).unwrap())
.unwrap()
.public()
.to_ss58check()
.contains("ab")
);
assert!(sr25519::Pair::from_seed_slice(&hex::decode(&seed[2..]).unwrap())
.unwrap()
.public()
.to_ss58check()
.contains("ab"));
}
#[test]
fn generate_key_respects_network_override() {
let seed = generate_key::<sr25519::Pair>("ab", Ss58AddressFormat::PolkadotAccount).unwrap();
assert!(
sr25519::Pair::from_seed_slice(&hex::decode(&seed[2..]).unwrap())
.unwrap()
.public()
.to_ss58check_with_version(Ss58AddressFormat::PolkadotAccount)
.contains("ab")
);
assert!(sr25519::Pair::from_seed_slice(&hex::decode(&seed[2..]).unwrap())
.unwrap()
.public()
.to_ss58check_with_version(Ss58AddressFormat::PolkadotAccount)
.contains("ab"));
}
#[test]
@@ -208,10 +198,7 @@ mod tests {
#[test]
fn test_score_100() {
let score = calculate_score(
"Polkadot",
"5PolkadotwHY5k9GpdTgpqs9xjuNvtv8EcwCFpEeyEf3KHim",
);
let score = calculate_score("Polkadot", "5PolkadotwHY5k9GpdTgpqs9xjuNvtv8EcwCFpEeyEf3KHim");
assert_eq!(score, 430);
}
@@ -219,10 +206,7 @@ mod tests {
fn test_score_50_2() {
// 50% for the position + 50% for the size
assert_eq!(
calculate_score(
"Polkadot",
"5PolkXXXXwHY5k9GpdTgpqs9xjuNvtv8EcwCFpEeyEf3KHim"
),
calculate_score("Polkadot", "5PolkXXXXwHY5k9GpdTgpqs9xjuNvtv8EcwCFpEeyEf3KHim"),
238
);
}
@@ -230,10 +214,7 @@ mod tests {
#[test]
fn test_score_0() {
assert_eq!(
calculate_score(
"Polkadot",
"5GUWv4bLCchGUHJrzULXnh4JgXsMpTKRnjuXTY7Qo1Kh9uYK"
),
calculate_score("Polkadot", "5GUWv4bLCchGUHJrzULXnh4JgXsMpTKRnjuXTY7Qo1Kh9uYK"),
0
);
}
+11 -20
View File
@@ -5,7 +5,7 @@
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
@@ -19,7 +19,7 @@
//! implementation of the `verify` subcommand
use crate::{error, utils, with_crypto_scheme, CryptoSchemeFlag};
use sp_core::{Public, crypto::Ss58Codec};
use sp_core::{crypto::Ss58Codec, Public};
use structopt::StructOpt;
/// The `verify` command
@@ -57,32 +57,23 @@ impl VerifyCmd {
let message = utils::read_message(self.message.as_ref(), self.hex)?;
let sig_data = utils::decode_hex(&self.sig)?;
let uri = utils::read_uri(self.uri.as_ref())?;
let uri = if uri.starts_with("0x") {
&uri[2..]
} else {
&uri
};
let uri = if uri.starts_with("0x") { &uri[2..] } else { &uri };
with_crypto_scheme!(
self.crypto_scheme.scheme,
verify(sig_data, message, uri)
)
with_crypto_scheme!(self.crypto_scheme.scheme, verify(sig_data, message, uri))
}
}
fn verify<Pair>(sig_data: Vec<u8>, message: Vec<u8>, uri: &str) -> error::Result<()>
where
Pair: sp_core::Pair,
Pair::Signature: Default + AsMut<[u8]>,
where
Pair: sp_core::Pair,
Pair::Signature: Default + AsMut<[u8]>,
{
let mut signature = Pair::Signature::default();
if sig_data.len() != signature.as_ref().len() {
return Err(
error::Error::SignatureInvalidLength {
read: sig_data.len(),
expected: signature.as_ref().len(),
}
);
return Err(error::Error::SignatureInvalidLength {
read: sig_data.len(),
expected: signature.as_ref().len(),
})
}
signature.as_mut().copy_from_slice(&sig_data);
+22 -42
View File
@@ -18,24 +18,24 @@
//! Configuration trait for a CLI based on substrate
use crate::arg_enums::Database;
use crate::error::Result;
use crate::{
DatabaseParams, ImportParams, KeystoreParams, NetworkParams, NodeKeyParams,
OffchainWorkerParams, PruningParams, SharedParams, SubstrateCli,
arg_enums::Database, error::Result, DatabaseParams, ImportParams, KeystoreParams,
NetworkParams, NodeKeyParams, OffchainWorkerParams, PruningParams, SharedParams, SubstrateCli,
};
use log::warn;
use names::{Generator, Name};
use sc_client_api::execution_extensions::ExecutionStrategies;
use sc_service::config::{
BasePath, Configuration, DatabaseConfig, ExtTransport, KeystoreConfig, NetworkConfiguration,
NodeKeyConfig, OffchainWorkerConfig, PrometheusConfig, PruningMode, Role, RpcMethods,
TaskExecutor, TelemetryEndpoints, TransactionPoolOptions, WasmExecutionMethod,
use sc_service::{
config::{
BasePath, Configuration, DatabaseConfig, ExtTransport, KeystoreConfig,
NetworkConfiguration, NodeKeyConfig, OffchainWorkerConfig, PrometheusConfig, PruningMode,
Role, RpcMethods, TaskExecutor, TelemetryEndpoints, TransactionPoolOptions,
WasmExecutionMethod,
},
ChainSpec, KeepBlocks, TracingReceiver, TransactionStorageMode,
};
use sc_service::{ChainSpec, TracingReceiver, KeepBlocks, TransactionStorageMode};
use sc_tracing::logging::LoggerBuilder;
use std::net::SocketAddr;
use std::path::PathBuf;
use std::{net::SocketAddr, path::PathBuf};
/// The maximum number of characters for a node name.
pub(crate) const NODE_NAME_MAX_LENGTH: usize = 64;
@@ -178,12 +178,7 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
default_listen_port,
)
} else {
NetworkConfiguration::new(
node_name,
client_id,
node_key,
Some(net_config_dir),
)
NetworkConfiguration::new(node_name, client_id, node_key, Some(net_config_dir))
})
}
@@ -201,14 +196,13 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
///
/// By default this is retrieved from `DatabaseParams` if it is available. Otherwise its `None`.
fn database_cache_size(&self) -> Result<Option<usize>> {
Ok(self.database_params()
.map(|x| x.database_cache_size())
.unwrap_or_default())
Ok(self.database_params().map(|x| x.database_cache_size()).unwrap_or_default())
}
/// Get the database transaction storage scheme.
fn database_transaction_storage(&self) -> Result<TransactionStorageMode> {
Ok(self.database_params()
Ok(self
.database_params()
.map(|x| x.transaction_storage())
.unwrap_or(TransactionStorageMode::BlockBody))
}
@@ -228,13 +222,8 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
database: Database,
) -> Result<DatabaseConfig> {
Ok(match database {
Database::RocksDb => DatabaseConfig::RocksDb {
path: base_path.join("db"),
cache_size,
},
Database::ParityDb => DatabaseConfig::ParityDb {
path: base_path.join("paritydb"),
},
Database::RocksDb => DatabaseConfig::RocksDb { path: base_path.join("db"), cache_size },
Database::ParityDb => DatabaseConfig::ParityDb { path: base_path.join("paritydb") },
})
}
@@ -242,9 +231,7 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
///
/// By default this is retrieved from `ImportParams` if it is available. Otherwise its `0`.
fn state_cache_size(&self) -> Result<usize> {
Ok(self.import_params()
.map(|x| x.state_cache_size())
.unwrap_or_default())
Ok(self.import_params().map(|x| x.state_cache_size()).unwrap_or_default())
}
/// Get the state cache child ratio (if any).
@@ -293,18 +280,14 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
/// By default this is retrieved from `ImportParams` if it is available. Otherwise its
/// `WasmExecutionMethod::default()`.
fn wasm_method(&self) -> Result<WasmExecutionMethod> {
Ok(self.import_params()
.map(|x| x.wasm_method())
.unwrap_or_default())
Ok(self.import_params().map(|x| x.wasm_method()).unwrap_or_default())
}
/// Get the path where WASM overrides live.
///
/// By default this is `None`.
fn wasm_runtime_overrides(&self) -> Option<PathBuf> {
self.import_params()
.map(|x| x.wasm_runtime_overrides())
.unwrap_or_default()
self.import_params().map(|x| x.wasm_runtime_overrides()).unwrap_or_default()
}
/// Get the execution strategies.
@@ -502,10 +485,7 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
let (keystore_remote, keystore) = self.keystore_config(&config_dir)?;
let telemetry_endpoints = self.telemetry_endpoints(&chain_spec)?;
let unsafe_pruning = self
.import_params()
.map(|p| p.unsafe_pruning)
.unwrap_or(false);
let unsafe_pruning = self.import_params().map(|p| p.unsafe_pruning).unwrap_or(false);
Ok(Configuration {
impl_name: C::impl_name(),
@@ -628,7 +608,7 @@ pub fn generate_node_name() -> String {
let count = node_name.chars().count();
if count < NODE_NAME_MAX_LENGTH {
return node_name;
return node_name
}
}
}
+1 -1
View File
@@ -159,7 +159,7 @@ pub trait SubstrateCli: Sized {
let _ = std::io::stdout().write_all(e.message.as_bytes());
std::process::exit(0);
}
}
},
};
<Self as StructOpt>::from_clap(&matches)
@@ -17,8 +17,8 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::arg_enums::Database;
use structopt::StructOpt;
use sc_service::TransactionStorageMode;
use structopt::StructOpt;
/// Parameters for block import.
#[derive(Debug, StructOpt, Clone)]
@@ -16,16 +16,17 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::arg_enums::{
ExecutionStrategy, WasmExecutionMethod, DEFAULT_EXECUTION_BLOCK_CONSTRUCTION,
DEFAULT_EXECUTION_IMPORT_BLOCK, DEFAULT_EXECUTION_IMPORT_BLOCK_VALIDATOR,
DEFAULT_EXECUTION_OFFCHAIN_WORKER, DEFAULT_EXECUTION_OTHER, DEFAULT_EXECUTION_SYNCING,
use crate::{
arg_enums::{
ExecutionStrategy, WasmExecutionMethod, DEFAULT_EXECUTION_BLOCK_CONSTRUCTION,
DEFAULT_EXECUTION_IMPORT_BLOCK, DEFAULT_EXECUTION_IMPORT_BLOCK_VALIDATOR,
DEFAULT_EXECUTION_OFFCHAIN_WORKER, DEFAULT_EXECUTION_OTHER, DEFAULT_EXECUTION_SYNCING,
},
params::{DatabaseParams, PruningParams},
};
use crate::params::DatabaseParams;
use crate::params::PruningParams;
use sc_client_api::execution_extensions::ExecutionStrategies;
use structopt::StructOpt;
use std::path::PathBuf;
use structopt::StructOpt;
#[cfg(feature = "wasmtime")]
const WASM_METHOD_DEFAULT: &str = "Compiled";
@@ -73,11 +74,7 @@ pub struct ImportParams {
pub execution_strategies: ExecutionStrategiesParams,
/// Specify the state cache size.
#[structopt(
long = "state-cache-size",
value_name = "Bytes",
default_value = "67108864"
)]
#[structopt(long = "state-cache-size", value_name = "Bytes", default_value = "67108864")]
pub state_cache_size: usize,
}
@@ -102,11 +99,7 @@ impl ImportParams {
pub fn execution_strategies(&self, is_dev: bool, is_validator: bool) -> ExecutionStrategies {
let exec = &self.execution_strategies;
let exec_all_or = |strat: Option<ExecutionStrategy>, default: ExecutionStrategy| {
let default = if is_dev {
ExecutionStrategy::Native
} else {
default
};
let default = if is_dev { ExecutionStrategy::Native } else { default };
exec.execution.unwrap_or_else(|| strat.unwrap_or(default)).into()
};
@@ -120,10 +113,14 @@ impl ImportParams {
ExecutionStrategies {
syncing: exec_all_or(exec.execution_syncing, DEFAULT_EXECUTION_SYNCING),
importing: exec_all_or(exec.execution_import_block, default_execution_import_block),
block_construction:
exec_all_or(exec.execution_block_construction, DEFAULT_EXECUTION_BLOCK_CONSTRUCTION),
offchain_worker:
exec_all_or(exec.execution_offchain_worker, DEFAULT_EXECUTION_OFFCHAIN_WORKER),
block_construction: exec_all_or(
exec.execution_block_construction,
DEFAULT_EXECUTION_BLOCK_CONSTRUCTION,
),
offchain_worker: exec_all_or(
exec.execution_offchain_worker,
DEFAULT_EXECUTION_OFFCHAIN_WORKER,
),
other: exec_all_or(exec.execution_other, DEFAULT_EXECUTION_OTHER),
}
}
@@ -16,12 +16,14 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::error::Result;
use crate::{error, error::Result};
use sc_service::config::KeystoreConfig;
use std::{fs, path::{PathBuf, Path}};
use structopt::StructOpt;
use crate::error;
use sp_core::crypto::SecretString;
use std::{
fs,
path::{Path, PathBuf},
};
use structopt::StructOpt;
/// default sub directory for the key store
const DEFAULT_KEYSTORE_CONFIG_PATH: &'static str = "keystore";
@@ -81,8 +83,7 @@ impl KeystoreParams {
#[cfg(target_os = "unknown")]
None
} else if let Some(ref file) = self.password_filename {
let password = fs::read_to_string(file)
.map_err(|e| format!("{}", e))?;
let password = fs::read_to_string(file).map_err(|e| format!("{}", e))?;
Some(SecretString::new(password))
} else {
self.password.clone()
+16 -21
View File
@@ -25,21 +25,20 @@ mod pruning_params;
mod shared_params;
mod transaction_pool_params;
use std::{fmt::Debug, str::FromStr, convert::TryFrom};
use sp_runtime::{generic::BlockId, traits::{Block as BlockT, NumberFor}};
use crate::arg_enums::{CryptoScheme, OutputType};
use sp_core::crypto::Ss58AddressFormat;
use crate::arg_enums::{OutputType, CryptoScheme};
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, NumberFor},
};
use std::{convert::TryFrom, fmt::Debug, str::FromStr};
use structopt::StructOpt;
pub use crate::params::database_params::*;
pub use crate::params::import_params::*;
pub use crate::params::keystore_params::*;
pub use crate::params::network_params::*;
pub use crate::params::node_key_params::*;
pub use crate::params::offchain_worker_params::*;
pub use crate::params::pruning_params::*;
pub use crate::params::shared_params::*;
pub use crate::params::transaction_pool_params::*;
pub use crate::params::{
database_params::*, import_params::*, keystore_params::*, network_params::*,
node_key_params::*, offchain_worker_params::*, pruning_params::*, shared_params::*,
transaction_pool_params::*,
};
/// Wrapper type of `String` that holds an unsigned integer of arbitrary size, formatted as a decimal.
#[derive(Debug, Clone)]
@@ -50,10 +49,7 @@ impl FromStr for GenericNumber {
fn from_str(block_number: &str) -> Result<Self, Self::Err> {
if let Some(pos) = block_number.chars().position(|d| !d.is_digit(10)) {
Err(format!(
"Expected block number, found illegal digit at position: {}",
pos,
))
Err(format!("Expected block number, found illegal digit at position: {}", pos,))
} else {
Ok(Self(block_number.to_owned()))
}
@@ -66,9 +62,9 @@ impl GenericNumber {
/// See `https://doc.rust-lang.org/std/primitive.str.html#method.parse` for more elaborate
/// documentation.
pub fn parse<N>(&self) -> Result<N, String>
where
N: FromStr,
N::Err: std::fmt::Debug,
where
N: FromStr,
N::Err: std::fmt::Debug,
{
FromStr::from_str(&self.0).map_err(|e| format!("Failed to parse block number: {:?}", e))
}
@@ -109,7 +105,7 @@ impl BlockNumberOrHash {
if self.0.starts_with("0x") {
Ok(BlockId::Hash(
FromStr::from_str(&self.0[2..])
.map_err(|e| format!("Failed to parse block hash: {:?}", e))?
.map_err(|e| format!("Failed to parse block hash: {:?}", e))?,
))
} else {
GenericNumber(self.0.clone()).parse().map(BlockId::Number)
@@ -117,7 +113,6 @@ impl BlockNumberOrHash {
}
}
/// Optional flag for specifying crypto algorithm
#[derive(Debug, StructOpt, Clone)]
pub struct CryptoSchemeFlag {
@@ -16,13 +16,17 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::params::node_key_params::NodeKeyParams;
use crate::arg_enums::SyncMode;
use crate::{arg_enums::SyncMode, params::node_key_params::NodeKeyParams};
use sc_network::{
config::{NetworkConfiguration, NodeKeyConfig, NonReservedPeerMode, SetConfig, TransportConfig},
config::{
NetworkConfiguration, NodeKeyConfig, NonReservedPeerMode, SetConfig, TransportConfig,
},
multiaddr::Protocol,
};
use sc_service::{ChainSpec, ChainType, config::{Multiaddr, MultiaddrWithPeerId}};
use sc_service::{
config::{Multiaddr, MultiaddrWithPeerId},
ChainSpec, ChainType,
};
use std::{borrow::Cow, path::PathBuf};
use structopt::StructOpt;
@@ -97,11 +101,7 @@ pub struct NetworkParams {
///
/// This allows downloading announced blocks from multiple peers. Decrease to save
/// traffic and risk increased latency.
#[structopt(
long = "max-parallel-downloads",
value_name = "COUNT",
default_value = "5"
)]
#[structopt(long = "max-parallel-downloads", value_name = "COUNT", default_value = "5")]
pub max_parallel_downloads: u32,
#[allow(missing_docs)]
@@ -184,15 +184,16 @@ impl NetworkParams {
let chain_type = chain_spec.chain_type();
// Activate if the user explicitly requested local discovery, `--dev` is given or the
// chain type is `Local`/`Development`
let allow_non_globals_in_dht = self.discover_local
|| is_dev
|| matches!(chain_type, ChainType::Local | ChainType::Development);
let allow_non_globals_in_dht =
self.discover_local ||
is_dev || matches!(chain_type, ChainType::Local | ChainType::Development);
let allow_private_ipv4 = match (self.allow_private_ipv4, self.no_private_ipv4) {
(true, true) => unreachable!("`*_private_ipv4` flags are mutually exclusive; qed"),
(true, false) => true,
(false, true) => false,
(false, false) => is_dev || matches!(chain_type, ChainType::Local | ChainType::Development),
(false, false) =>
is_dev || matches!(chain_type, ChainType::Local | ChainType::Development),
};
NetworkConfiguration {
@@ -16,13 +16,12 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use sc_network::{config::identity::ed25519, config::NodeKeyConfig};
use sc_network::config::{identity::ed25519, NodeKeyConfig};
use sp_core::H256;
use std::{path::PathBuf, str::FromStr};
use structopt::StructOpt;
use crate::arg_enums::NodeKeyType;
use crate::error;
use crate::{arg_enums::NodeKeyType, error};
/// The file name of the node's Ed25519 secret key inside the chain-specific
/// network config directory, if neither `--node-key` nor `--node-key-file`
@@ -103,12 +102,12 @@ impl NodeKeyParams {
sc_network::config::Secret::File(
self.node_key_file
.clone()
.unwrap_or_else(|| net_config_dir.join(NODE_KEY_ED25519_FILE))
.unwrap_or_else(|| net_config_dir.join(NODE_KEY_ED25519_FILE)),
)
};
NodeKeyConfig::Ed25519(secret)
}
},
})
}
}
@@ -120,13 +119,11 @@ fn invalid_node_key(e: impl std::fmt::Display) -> error::Error {
/// Parse a Ed25519 secret key from a hex string into a `sc_network::Secret`.
fn parse_ed25519_secret(hex: &str) -> error::Result<sc_network::config::Ed25519Secret> {
H256::from_str(&hex)
.map_err(invalid_node_key)
.and_then(|bytes| {
ed25519::SecretKey::from_bytes(bytes)
.map(sc_network::config::Secret::Input)
.map_err(invalid_node_key)
})
H256::from_str(&hex).map_err(invalid_node_key).and_then(|bytes| {
ed25519::SecretKey::from_bytes(bytes)
.map(sc_network::config::Secret::Input)
.map_err(invalid_node_key)
})
}
#[cfg(test)]
@@ -151,9 +148,7 @@ mod tests {
params.node_key(net_config_dir).and_then(|c| match c {
NodeKeyConfig::Ed25519(sc_network::config::Secret::Input(ref ski))
if node_key_type == NodeKeyType::Ed25519 && &sk[..] == ski.as_ref() =>
{
Ok(())
}
Ok(()),
_ => Err(error::Error::Input("Unexpected node key config".into())),
})
})
@@ -171,14 +166,14 @@ mod tests {
node_key_file: Some(file),
};
let node_key = params.node_key(&PathBuf::from("not-used"))
let node_key = params
.node_key(&PathBuf::from("not-used"))
.expect("Creates node key config")
.into_keypair()
.expect("Creates node key pair");
match node_key {
Keypair::Ed25519(ref pair)
if pair.secret().as_ref() == key.as_ref() => {}
Keypair::Ed25519(ref pair) if pair.secret().as_ref() == key.as_ref() => {},
_ => panic!("Invalid key"),
}
}
@@ -202,11 +197,7 @@ mod tests {
{
NodeKeyType::variants().iter().try_for_each(|t| {
let node_key_type = NodeKeyType::from_str(t).unwrap();
f(NodeKeyParams {
node_key_type,
node_key: None,
node_key_file: None,
})
f(NodeKeyParams { node_key_type, node_key: None, node_key_file: None })
})
}
@@ -214,17 +205,12 @@ mod tests {
with_def_params(|params| {
let dir = PathBuf::from(net_config_dir.clone());
let typ = params.node_key_type;
params
.node_key(net_config_dir)
.and_then(move |c| match c {
NodeKeyConfig::Ed25519(sc_network::config::Secret::File(ref f))
if typ == NodeKeyType::Ed25519
&& f == &dir.join(NODE_KEY_ED25519_FILE) =>
{
Ok(())
}
_ => Err(error::Error::Input("Unexpected node key config".into())),
})
params.node_key(net_config_dir).and_then(move |c| match c {
NodeKeyConfig::Ed25519(sc_network::config::Secret::File(ref f))
if typ == NodeKeyType::Ed25519 && f == &dir.join(NODE_KEY_ED25519_FILE) =>
Ok(()),
_ => Err(error::Error::Input("Unexpected node key config".into())),
})
})
}
@@ -27,8 +27,7 @@ use sc_network::config::Role;
use sc_service::config::OffchainWorkerConfig;
use structopt::StructOpt;
use crate::error;
use crate::OffchainWorkerEnabled;
use crate::{error, OffchainWorkerEnabled};
/// Offchain worker related parameters.
#[derive(Debug, StructOpt, Clone)]
@@ -49,10 +48,7 @@ pub struct OffchainWorkerParams {
///
/// Enables a runtime to write directly to a offchain workers
/// DB during block import.
#[structopt(
long = "enable-offchain-indexing",
value_name = "ENABLE_OFFCHAIN_INDEXING"
)]
#[structopt(long = "enable-offchain-indexing", value_name = "ENABLE_OFFCHAIN_INDEXING")]
pub indexing_enabled: bool,
}
@@ -67,9 +63,6 @@ impl OffchainWorkerParams {
};
let indexing_enabled = self.indexing_enabled;
Ok(OffchainWorkerConfig {
enabled,
indexing_enabled,
})
Ok(OffchainWorkerConfig { enabled, indexing_enabled })
}
}
@@ -17,7 +17,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::error;
use sc_service::{PruningMode, Role, KeepBlocks};
use sc_service::{KeepBlocks, PruningMode, Role};
use structopt::StructOpt;
/// Parameters to define the pruning mode
@@ -54,13 +54,13 @@ impl PruningParams {
"Validators should run with state pruning disabled (i.e. archive). \
You can ignore this check with `--unsafe-pruning`."
.to_string(),
));
))
}
PruningMode::keep_blocks(s.parse().map_err(|_| {
error::Error::Input("Invalid pruning mode specified".to_string())
})?)
}
},
})
}
@@ -16,10 +16,10 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::arg_enums::TracingReceiver;
use sc_service::config::BasePath;
use std::path::PathBuf;
use structopt::StructOpt;
use crate::arg_enums::TracingReceiver;
/// Shared parameters used by all `CoreParams`.
#[derive(Debug, StructOpt, Clone)]
@@ -88,13 +88,12 @@ impl SharedParams {
pub fn chain_id(&self, is_dev: bool) -> String {
match self.chain {
Some(ref chain) => chain.clone(),
None => {
None =>
if is_dev {
"dev".into()
} else {
"".into()
}
}
},
}
}
+21 -36
View File
@@ -16,19 +16,13 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::CliConfiguration;
use crate::Result;
use crate::SubstrateCli;
use crate::{error::Error as CliError, CliConfiguration, Result, SubstrateCli};
use chrono::prelude::*;
use futures::pin_mut;
use futures::select;
use futures::{future, future::FutureExt, Future};
use futures::{future, future::FutureExt, pin_mut, select, Future};
use log::info;
use sc_service::{Configuration, TaskType, TaskManager};
use sc_service::{Configuration, Error as ServiceError, TaskManager, TaskType};
use sp_utils::metrics::{TOKIO_THREADS_ALIVE, TOKIO_THREADS_TOTAL};
use std::marker::PhantomData;
use sc_service::Error as ServiceError;
use crate::error::Error as CliError;
#[cfg(target_family = "unix")]
async fn main<F, E>(func: F) -> std::result::Result<(), E>
@@ -119,27 +113,19 @@ pub struct Runner<C: SubstrateCli> {
impl<C: SubstrateCli> Runner<C> {
/// Create a new runtime with the command provided in argument
pub fn new<T: CliConfiguration>(
cli: &C,
command: &T,
) -> Result<Runner<C>> {
pub fn new<T: CliConfiguration>(cli: &C, command: &T) -> Result<Runner<C>> {
let tokio_runtime = build_runtime()?;
let runtime_handle = tokio_runtime.handle().clone();
let task_executor = move |fut, task_type| {
match task_type {
TaskType::Async => runtime_handle.spawn(fut).map(drop),
TaskType::Blocking =>
runtime_handle.spawn_blocking(move || futures::executor::block_on(fut))
.map(drop),
}
let task_executor = move |fut, task_type| match task_type {
TaskType::Async => runtime_handle.spawn(fut).map(drop),
TaskType::Blocking => runtime_handle
.spawn_blocking(move || futures::executor::block_on(fut))
.map(drop),
};
Ok(Runner {
config: command.create_configuration(
cli,
task_executor.into(),
)?,
config: command.create_configuration(cli, task_executor.into())?,
tokio_runtime,
phantom: PhantomData,
})
@@ -183,7 +169,7 @@ impl<C: SubstrateCli> Runner<C> {
/// A helper function that runs a command with the configuration of this node.
pub fn sync_run<E>(
self,
runner: impl FnOnce(Configuration) -> std::result::Result<(), E>
runner: impl FnOnce(Configuration) -> std::result::Result<(), E>,
) -> std::result::Result<(), E>
where
E: std::error::Error + Send + Sync + 'static + From<ServiceError>,
@@ -194,7 +180,8 @@ impl<C: SubstrateCli> Runner<C> {
/// A helper function that runs a future with tokio and stops if the process receives
/// the signal `SIGTERM` or `SIGINT`.
pub fn async_run<F, E>(
self, runner: impl FnOnce(Configuration) -> std::result::Result<(F, TaskManager), E>,
self,
runner: impl FnOnce(Configuration) -> std::result::Result<(F, TaskManager), E>,
) -> std::result::Result<(), E>
where
F: Future<Output = std::result::Result<(), E>>,
@@ -219,19 +206,17 @@ impl<C: SubstrateCli> Runner<C> {
pub fn print_node_infos<C: SubstrateCli>(config: &Configuration) {
info!("{}", C::impl_name());
info!("✌️ version {}", C::impl_version());
info!(
"❤️ by {}, {}-{}",
C::author(),
C::copyright_start_year(),
Local::today().year(),
);
info!("❤️ by {}, {}-{}", C::author(), C::copyright_start_year(), Local::today().year(),);
info!("📋 Chain specification: {}", config.chain_spec.name());
info!("🏷 Node name: {}", config.network.node_name);
info!("👤 Role: {}", config.display_role());
info!("💾 Database: {} at {}",
config.database,
config.database.path().map_or_else(|| "<unknown>".to_owned(), |p| p.display().to_string())
info!(
"💾 Database: {} at {}",
config.database,
config
.database
.path()
.map_or_else(|| "<unknown>".to_owned(), |p| p.display().to_string())
);
info!("⛓ Native runtime: {}", C::native_runtime_version(&config.chain_spec));
}
@@ -18,36 +18,37 @@
//! Module implementing the logic for verifying and importing AuRa blocks.
use crate::{AuthorityId, find_pre_digest, slot_author, aura_err, Error, authorities};
use std::{
sync::Arc, marker::PhantomData, hash::Hash, fmt::Debug,
};
use crate::{aura_err, authorities, find_pre_digest, slot_author, AuthorityId, Error};
use codec::{Codec, Decode, Encode};
use log::{debug, info, trace};
use prometheus_endpoint::Registry;
use codec::{Encode, Decode, Codec};
use sp_consensus::{
BlockImport, CanAuthorWith, ForkChoiceStrategy, BlockImportParams,
BlockOrigin, Error as ConsensusError,
import_queue::{
Verifier, BasicQueue, DefaultImportQueue, BoxJustificationImport,
},
};
use sc_client_api::{BlockOf, UsageProvider, backend::AuxStore};
use sp_blockchain::{well_known_cache_keys::{self, Id as CacheKeyId}, ProvideCache, HeaderBackend};
use sc_client_api::{backend::AuxStore, BlockOf, UsageProvider};
use sc_consensus_slots::{check_equivocation, CheckedHeader, InherentDataProviderExt};
use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_TRACE};
use sp_api::{ApiExt, ProvideRuntimeApi};
use sp_block_builder::BlockBuilder as BlockBuilderApi;
use sp_runtime::{generic::{BlockId, OpaqueDigestItemId}, Justifications};
use sp_runtime::traits::{Block as BlockT, Header, DigestItemFor};
use sp_api::ProvideRuntimeApi;
use sp_blockchain::{
well_known_cache_keys::{self, Id as CacheKeyId},
HeaderBackend, ProvideCache,
};
use sp_consensus::{
import_queue::{BasicQueue, BoxJustificationImport, DefaultImportQueue, Verifier},
BlockImport, BlockImportParams, BlockOrigin, CanAuthorWith, Error as ConsensusError,
ForkChoiceStrategy,
};
use sp_consensus_aura::{
digests::CompatibleDigestItem, inherents::AuraInherentData, AuraApi, ConsensusLog,
AURA_ENGINE_ID,
};
use sp_consensus_slots::Slot;
use sp_core::crypto::Pair;
use sp_inherents::{CreateInherentDataProviders, InherentDataProvider as _};
use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_TRACE, CONSENSUS_DEBUG};
use sc_consensus_slots::{CheckedHeader, check_equivocation, InherentDataProviderExt};
use sp_consensus_slots::Slot;
use sp_api::ApiExt;
use sp_consensus_aura::{
digests::CompatibleDigestItem, AuraApi, inherents::AuraInherentData,
ConsensusLog, AURA_ENGINE_ID,
use sp_runtime::{
generic::{BlockId, OpaqueDigestItemId},
traits::{Block as BlockT, DigestItemFor, Header},
Justifications,
};
use std::{fmt::Debug, hash::Hash, marker::PhantomData, sync::Arc};
/// check a header has been signed by the right key. If the slot is too far in the future, an error
/// will be returned. If it's successful, returns the pre-header and the digest item
@@ -61,7 +62,8 @@ fn check_header<C, B: BlockT, P: Pair>(
hash: B::Hash,
authorities: &[AuthorityId<P>],
check_for_equivocation: CheckForEquivocation,
) -> Result<CheckedHeader<B::Header, (Slot, DigestItemFor<B>)>, Error<B>> where
) -> Result<CheckedHeader<B::Header, (Slot, DigestItemFor<B>)>, Error<B>>
where
DigestItemFor<B>: CompatibleDigestItem<P::Signature>,
P::Signature: Codec,
C: sc_client_api::backend::AuxStore,
@@ -69,9 +71,7 @@ fn check_header<C, B: BlockT, P: Pair>(
{
let seal = header.digest_mut().pop().ok_or_else(|| Error::HeaderUnsealed(hash))?;
let sig = seal.as_aura_seal().ok_or_else(|| {
aura_err(Error::HeaderBadSeal(hash))
})?;
let sig = seal.as_aura_seal().ok_or_else(|| aura_err(Error::HeaderBadSeal(hash)))?;
let slot = find_pre_digest::<B, P::Signature>(&header)?;
@@ -81,20 +81,17 @@ fn check_header<C, B: BlockT, P: Pair>(
} else {
// check the signature is valid under the expected authority and
// chain state.
let expected_author = slot_author::<P>(slot, &authorities)
.ok_or_else(|| Error::SlotAuthorNotFound)?;
let expected_author =
slot_author::<P>(slot, &authorities).ok_or_else(|| Error::SlotAuthorNotFound)?;
let pre_hash = header.hash();
if P::verify(&sig, pre_hash.as_ref(), expected_author) {
if check_for_equivocation.check_for_equivocation() {
if let Some(equivocation_proof) = check_equivocation(
client,
slot_now,
slot,
&header,
expected_author,
).map_err(Error::Client)? {
if let Some(equivocation_proof) =
check_equivocation(client, slot_now, slot, &header, expected_author)
.map_err(Error::Client)?
{
info!(
target: "aura",
"Slot author is equivocating at slot {} with headers {:?} and {:?}",
@@ -141,7 +138,8 @@ impl<C, P, CAW, CIDP> AuraVerifier<C, P, CAW, CIDP> {
}
}
impl<C, P, CAW, CIDP> AuraVerifier<C, P, CAW, CIDP> where
impl<C, P, CAW, CIDP> AuraVerifier<C, P, CAW, CIDP>
where
P: Send + Sync + 'static,
CAW: Send + Sync + 'static,
CIDP: Send,
@@ -152,8 +150,10 @@ impl<C, P, CAW, CIDP> AuraVerifier<C, P, CAW, CIDP> where
block_id: BlockId<B>,
inherent_data: sp_inherents::InherentData,
create_inherent_data_providers: CIDP::InherentDataProviders,
) -> Result<(), Error<B>> where
C: ProvideRuntimeApi<B>, C::Api: BlockBuilderApi<B>,
) -> Result<(), Error<B>>
where
C: ProvideRuntimeApi<B>,
C::Api: BlockBuilderApi<B>,
CAW: CanAuthorWith<B>,
CIDP: CreateInherentDataProviders<B, ()>,
{
@@ -167,11 +167,11 @@ impl<C, P, CAW, CIDP> AuraVerifier<C, P, CAW, CIDP> where
return Ok(())
}
let inherent_res = self.client.runtime_api().check_inherents(
&block_id,
block,
inherent_data,
).map_err(|e| Error::Client(e.into()))?;
let inherent_res = self
.client
.runtime_api()
.check_inherents(&block_id, block, inherent_data)
.map_err(|e| Error::Client(e.into()))?;
if !inherent_res.ok() {
for (i, e) in inherent_res.into_errors() {
@@ -187,13 +187,14 @@ impl<C, P, CAW, CIDP> AuraVerifier<C, P, CAW, CIDP> where
}
#[async_trait::async_trait]
impl<B: BlockT, C, P, CAW, CIDP> Verifier<B> for AuraVerifier<C, P, CAW, CIDP> where
C: ProvideRuntimeApi<B> +
Send +
Sync +
sc_client_api::backend::AuxStore +
ProvideCache<B> +
BlockOf,
impl<B: BlockT, C, P, CAW, CIDP> Verifier<B> for AuraVerifier<C, P, CAW, CIDP>
where
C: ProvideRuntimeApi<B>
+ Send
+ Sync
+ sc_client_api::backend::AuxStore
+ ProvideCache<B>
+ BlockOf,
C::Api: BlockBuilderApi<B> + AuraApi<B, AuthorityId<P>> + ApiExt<B>,
DigestItemFor<B>: CompatibleDigestItem<P::Signature>,
P: Pair + Send + Sync + 'static,
@@ -215,15 +216,14 @@ impl<B: BlockT, C, P, CAW, CIDP> Verifier<B> for AuraVerifier<C, P, CAW, CIDP> w
let authorities = authorities(self.client.as_ref(), &BlockId::Hash(parent_hash))
.map_err(|e| format!("Could not fetch authorities at {:?}: {:?}", parent_hash, e))?;
let create_inherent_data_providers = self.create_inherent_data_providers
.create_inherent_data_providers(
parent_hash,
(),
)
let create_inherent_data_providers = self
.create_inherent_data_providers
.create_inherent_data_providers(parent_hash, ())
.await
.map_err(|e| Error::<B>::Client(sp_blockchain::Error::Application(e)))?;
let mut inherent_data = create_inherent_data_providers.create_inherent_data()
let mut inherent_data = create_inherent_data_providers
.create_inherent_data()
.map_err(Error::<B>::Inherent)?;
let slot_now = create_inherent_data_providers.slot();
@@ -238,7 +238,8 @@ impl<B: BlockT, C, P, CAW, CIDP> Verifier<B> for AuraVerifier<C, P, CAW, CIDP> w
hash,
&authorities[..],
self.check_for_equivocation,
).map_err(|e| e.to_string())?;
)
.map_err(|e| e.to_string())?;
match checked_header {
CheckedHeader::Checked(pre_header, (slot, seal)) => {
// if the body is passed through, we need to use the runtime
@@ -250,7 +251,8 @@ impl<B: BlockT, C, P, CAW, CIDP> Verifier<B> for AuraVerifier<C, P, CAW, CIDP> w
inherent_data.aura_replace_inherent_data(slot);
// skip the inherents verification if the runtime API is old.
if self.client
if self
.client
.runtime_api()
.has_api_with::<dyn BlockBuilderApi<B>, _>(
&BlockId::Hash(parent_hash),
@@ -263,7 +265,9 @@ impl<B: BlockT, C, P, CAW, CIDP> Verifier<B> for AuraVerifier<C, P, CAW, CIDP> w
BlockId::Hash(parent_hash),
inherent_data,
create_inherent_data_providers,
).await.map_err(|e| e.to_string())?;
)
.await
.map_err(|e| e.to_string())?;
}
let (_, inner_body) = block.deconstruct();
@@ -279,16 +283,18 @@ impl<B: BlockT, C, P, CAW, CIDP> Verifier<B> for AuraVerifier<C, P, CAW, CIDP> w
);
// Look for an authorities-change log.
let maybe_keys = pre_header.digest()
let maybe_keys = pre_header
.digest()
.logs()
.iter()
.filter_map(|l| l.try_to::<ConsensusLog<AuthorityId<P>>>(
OpaqueDigestItemId::Consensus(&AURA_ENGINE_ID)
))
.filter_map(|l| {
l.try_to::<ConsensusLog<AuthorityId<P>>>(OpaqueDigestItemId::Consensus(
&AURA_ENGINE_ID,
))
})
.find_map(|l| match l {
ConsensusLog::AuthoritiesChange(a) => Some(
vec![(well_known_cache_keys::AUTHORITIES, a.encode())]
),
ConsensusLog::AuthoritiesChange(a) =>
Some(vec![(well_known_cache_keys::AUTHORITIES, a.encode())]),
_ => None,
});
@@ -300,7 +306,7 @@ impl<B: BlockT, C, P, CAW, CIDP> Verifier<B> for AuraVerifier<C, P, CAW, CIDP> w
import_block.post_hash = Some(hash);
Ok((import_block, maybe_keys))
}
},
CheckedHeader::Deferred(a, b) => {
debug!(target: "aura", "Checking {:?} failed; {:?}, {:?}.", hash, a, b);
telemetry!(
@@ -312,7 +318,7 @@ impl<B: BlockT, C, P, CAW, CIDP> Verifier<B> for AuraVerifier<C, P, CAW, CIDP> w
"b" => ?b,
);
Err(format!("Header {:?} rejected: too far in the future", hash))
}
},
}
}
}
@@ -375,8 +381,9 @@ pub fn import_queue<'a, P, Block, I, C, S, CAW, CIDP>(
can_author_with,
check_for_equivocation,
telemetry,
}: ImportQueueParams<'a, Block, I, C, S, CAW, CIDP>
) -> Result<DefaultImportQueue<Block, C>, sp_consensus::Error> where
}: ImportQueueParams<'a, Block, I, C, S, CAW, CIDP>,
) -> Result<DefaultImportQueue<Block, C>, sp_consensus::Error>
where
Block: BlockT,
C::Api: BlockBuilderApi<Block> + AuraApi<Block, AuthorityId<P>> + ApiExt<Block>,
C: 'static
@@ -388,7 +395,7 @@ pub fn import_queue<'a, P, Block, I, C, S, CAW, CIDP>(
+ AuxStore
+ UsageProvider<Block>
+ HeaderBackend<Block>,
I: BlockImport<Block, Error=ConsensusError, Transaction = sp_api::TransactionFor<C, Block>>
I: BlockImport<Block, Error = ConsensusError, Transaction = sp_api::TransactionFor<C, Block>>
+ Send
+ Sync
+ 'static,
@@ -401,23 +408,15 @@ pub fn import_queue<'a, P, Block, I, C, S, CAW, CIDP>(
CIDP: CreateInherentDataProviders<Block, ()> + Sync + Send + 'static,
CIDP::InherentDataProviders: InherentDataProviderExt + Send + Sync,
{
let verifier = build_verifier::<P, _, _, _>(
BuildVerifierParams {
client,
create_inherent_data_providers,
can_author_with,
check_for_equivocation,
telemetry,
},
);
let verifier = build_verifier::<P, _, _, _>(BuildVerifierParams {
client,
create_inherent_data_providers,
can_author_with,
check_for_equivocation,
telemetry,
});
Ok(BasicQueue::new(
verifier,
Box::new(block_import),
justification_import,
spawner,
registry,
))
Ok(BasicQueue::new(verifier, Box::new(block_import), justification_import, spawner, registry))
}
/// Parameters of [`build_verifier`].
@@ -442,7 +441,7 @@ pub fn build_verifier<P, C, CIDP, CAW>(
can_author_with,
check_for_equivocation,
telemetry,
}: BuildVerifierParams<C, CIDP, CAW>
}: BuildVerifierParams<C, CIDP, CAW>,
) -> AuraVerifier<C, P, CAW, CIDP> {
AuraVerifier::<_, P, _, _>::new(
client,
+192 -169
View File
@@ -31,50 +31,53 @@
//! NOTE: Aura itself is designed to be generic over the crypto used.
#![forbid(missing_docs, unsafe_code)]
use std::{
sync::Arc, marker::PhantomData, hash::Hash, fmt::Debug, pin::Pin,
convert::{TryFrom, TryInto},
fmt::Debug,
hash::Hash,
marker::PhantomData,
pin::Pin,
sync::Arc,
};
use futures::prelude::*;
use log::{debug, trace};
use codec::{Encode, Decode, Codec};
use codec::{Codec, Decode, Encode};
use sp_consensus::{
BlockImport, Environment, Proposer, CanAuthorWith, ForkChoiceStrategy, BlockImportParams,
BlockOrigin, Error as ConsensusError, SelectChain, StateAction,
};
use sc_client_api::{backend::AuxStore, BlockOf, UsageProvider};
use sp_blockchain::{Result as CResult, ProvideCache, HeaderBackend};
use sp_core::crypto::Public;
use sp_application_crypto::{AppKey, AppPublic};
use sp_runtime::{generic::BlockId, traits::NumberFor};
use sp_runtime::traits::{Block as BlockT, Header, DigestItemFor, Zero, Member};
use sp_api::ProvideRuntimeApi;
use sp_core::crypto::Pair;
use sp_keystore::{SyncCryptoStorePtr, SyncCryptoStore};
use sp_inherents::CreateInherentDataProviders;
use sc_telemetry::TelemetryHandle;
use sc_consensus_slots::{
SlotInfo, BackoffAuthoringBlocksStrategy, InherentDataProviderExt, StorageChanges,
BackoffAuthoringBlocksStrategy, InherentDataProviderExt, SlotInfo, StorageChanges,
};
use sc_telemetry::TelemetryHandle;
use sp_api::ProvideRuntimeApi;
use sp_application_crypto::{AppKey, AppPublic};
use sp_blockchain::{HeaderBackend, ProvideCache, Result as CResult};
use sp_consensus::{
BlockImport, BlockImportParams, BlockOrigin, CanAuthorWith, Environment,
Error as ConsensusError, ForkChoiceStrategy, Proposer, SelectChain, StateAction,
};
use sp_consensus_slots::Slot;
use sp_core::crypto::{Pair, Public};
use sp_inherents::CreateInherentDataProviders;
use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr};
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, DigestItemFor, Header, Member, NumberFor, Zero},
};
mod import_queue;
pub use sp_consensus_aura::{
ConsensusLog, AuraApi, AURA_ENGINE_ID, digests::CompatibleDigestItem,
inherents::{
InherentType as AuraInherent,
INHERENT_IDENTIFIER, InherentDataProvider,
},
};
pub use sp_consensus::SyncOracle;
pub use import_queue::{
ImportQueueParams, import_queue, CheckForEquivocation,
build_verifier, BuildVerifierParams, AuraVerifier,
build_verifier, import_queue, AuraVerifier, BuildVerifierParams, CheckForEquivocation,
ImportQueueParams,
};
pub use sc_consensus_slots::SlotProportion;
pub use sp_consensus::SyncOracle;
pub use sp_consensus_aura::{
digests::CompatibleDigestItem,
inherents::{InherentDataProvider, InherentType as AuraInherent, INHERENT_IDENTIFIER},
AuraApi, ConsensusLog, AURA_ENGINE_ID,
};
type AuthorityId<P> = <P as Pair>::Public;
@@ -82,7 +85,8 @@ type AuthorityId<P> = <P as Pair>::Public;
pub type SlotDuration = sc_consensus_slots::SlotDuration<sp_consensus_aura::SlotDuration>;
/// Get type of `SlotDuration` for Aura.
pub fn slot_duration<A, B, C>(client: &C) -> CResult<SlotDuration> where
pub fn slot_duration<A, B, C>(client: &C) -> CResult<SlotDuration>
where
A: Codec,
B: BlockT,
C: AuxStore + ProvideRuntimeApi<B> + UsageProvider<B>,
@@ -93,7 +97,9 @@ pub fn slot_duration<A, B, C>(client: &C) -> CResult<SlotDuration> where
/// Get slot author for given block along with authorities.
fn slot_author<P: Pair>(slot: Slot, authorities: &[AuthorityId<P>]) -> Option<&AuthorityId<P>> {
if authorities.is_empty() { return None }
if authorities.is_empty() {
return None
}
let idx = *slot % (authorities.len() as u64);
assert!(
@@ -101,9 +107,10 @@ fn slot_author<P: Pair>(slot: Slot, authorities: &[AuthorityId<P>]) -> Option<&A
"It is impossible to have a vector with length beyond the address space; qed",
);
let current_author = authorities.get(idx as usize)
.expect("authorities not empty; index constrained to list length;\
this is a valid index; qed");
let current_author = authorities.get(idx as usize).expect(
"authorities not empty; index constrained to list length;\
this is a valid index; qed",
);
Some(current_author)
}
@@ -325,9 +332,8 @@ where
type BlockImport = I;
type SyncOracle = SO;
type JustificationSyncLink = L;
type CreateProposer = Pin<Box<
dyn Future<Output = Result<E::Proposer, sp_consensus::Error>> + Send + 'static
>>;
type CreateProposer =
Pin<Box<dyn Future<Output = Result<E::Proposer, sp_consensus::Error>> + Send + 'static>>;
type Proposer = E::Proposer;
type Claim = P::Public;
type EpochData = Vec<AuthorityId<P>>;
@@ -376,22 +382,25 @@ where
slot: Slot,
_claim: &Self::Claim,
) -> Vec<sp_runtime::DigestItem<B::Hash>> {
vec![
<DigestItemFor<B> as CompatibleDigestItem<P::Signature>>::aura_pre_digest(slot),
]
vec![<DigestItemFor<B> as CompatibleDigestItem<P::Signature>>::aura_pre_digest(slot)]
}
fn block_import_params(&self) -> Box<dyn Fn(
B::Header,
&B::Hash,
Vec<B::Extrinsic>,
StorageChanges<sp_api::TransactionFor<C, B>, B>,
Self::Claim,
Self::EpochData,
) -> Result<
sp_consensus::BlockImportParams<B, sp_api::TransactionFor<C, B>>,
sp_consensus::Error> + Send + 'static>
{
fn block_import_params(
&self,
) -> Box<
dyn Fn(
B::Header,
&B::Hash,
Vec<B::Extrinsic>,
StorageChanges<sp_api::TransactionFor<C, B>, B>,
Self::Claim,
Self::EpochData,
) -> Result<
sp_consensus::BlockImportParams<B, sp_api::TransactionFor<C, B>>,
sp_consensus::Error,
> + Send
+ 'static,
> {
let keystore = self.keystore.clone();
Box::new(move |header, header_hash, body, storage_changes, public, _epoch| {
// sign the pre-sealed hash of the block and then
@@ -402,28 +411,28 @@ where
&*keystore,
<AuthorityId<P> as AppKey>::ID,
&public_type_pair,
header_hash.as_ref()
).map_err(|e| sp_consensus::Error::CannotSign(
public.clone(), e.to_string(),
))?
.ok_or_else(|| sp_consensus::Error::CannotSign(
public.clone(), "Could not find key in keystore.".into(),
))?;
let signature = signature.clone().try_into()
.map_err(|_| sp_consensus::Error::InvalidSignature(
signature, public
))?;
header_hash.as_ref(),
)
.map_err(|e| sp_consensus::Error::CannotSign(public.clone(), e.to_string()))?
.ok_or_else(|| {
sp_consensus::Error::CannotSign(
public.clone(),
"Could not find key in keystore.".into(),
)
})?;
let signature = signature
.clone()
.try_into()
.map_err(|_| sp_consensus::Error::InvalidSignature(signature, public))?;
let signature_digest_item = <
DigestItemFor<B> as CompatibleDigestItem<P::Signature>
>::aura_seal(signature);
let signature_digest_item =
<DigestItemFor<B> as CompatibleDigestItem<P::Signature>>::aura_seal(signature);
let mut import_block = BlockImportParams::new(BlockOrigin::Own, header);
import_block.post_digests.push(signature_digest_item);
import_block.body = Some(body);
import_block.state_action = StateAction::ApplyChanges(
sp_consensus::StorageChanges::Changes(storage_changes)
);
import_block.state_action =
StateAction::ApplyChanges(sp_consensus::StorageChanges::Changes(storage_changes));
import_block.fork_choice = Some(ForkChoiceStrategy::LongestChain);
Ok(import_block)
@@ -443,7 +452,7 @@ where
self.client.info().finalized_number,
slot,
self.logging_target(),
);
)
}
}
false
@@ -458,9 +467,11 @@ where
}
fn proposer(&mut self, block: &B::Header) -> Self::CreateProposer {
Box::pin(self.env.init(block).map_err(|e| {
sp_consensus::Error::ClientImport(format!("{:?}", e)).into()
}))
Box::pin(
self.env
.init(block)
.map_err(|e| sp_consensus::Error::ClientImport(format!("{:?}", e)).into()),
)
}
fn telemetry(&self) -> Option<TelemetryHandle> {
@@ -515,7 +526,7 @@ impl<B: BlockT> std::convert::From<Error<B>> for String {
fn find_pre_digest<B: BlockT, Signature: Codec>(header: &B::Header) -> Result<Slot, Error<B>> {
if header.number().is_zero() {
return Ok(0.into());
return Ok(0.into())
}
let mut pre_digest: Option<Slot> = None;
@@ -530,13 +541,15 @@ fn find_pre_digest<B: BlockT, Signature: Codec>(header: &B::Header) -> Result<Sl
pre_digest.ok_or_else(|| aura_err(Error::NoDigestFound))
}
fn authorities<A, B, C>(client: &C, at: &BlockId<B>) -> Result<Vec<A>, ConsensusError> where
fn authorities<A, B, C>(client: &C, at: &BlockId<B>) -> Result<Vec<A>, ConsensusError>
where
A: Codec + Debug,
B: BlockT,
C: ProvideRuntimeApi<B> + BlockOf + ProvideCache<B>,
C::Api: AuraApi<B, A>,
{
client.runtime_api()
client
.runtime_api()
.authorities(at)
.ok()
.ok_or_else(|| sp_consensus::Error::InvalidAuthoritiesSet.into())
@@ -545,26 +558,31 @@ fn authorities<A, B, C>(client: &C, at: &BlockId<B>) -> Result<Vec<A>, Consensus
#[cfg(test)]
mod tests {
use super::*;
use sp_consensus::{
NoNetwork as DummyOracle, Proposal, AlwaysCanAuthor, DisableProofRecording,
import_queue::BoxJustificationImport, SlotData,
};
use sc_network_test::{Block as TestBlock, *};
use sp_runtime::traits::{Block as BlockT, DigestFor};
use sc_network::config::ProtocolConfig;
use parking_lot::Mutex;
use sp_keyring::sr25519::Keyring;
use sc_client_api::BlockchainEvents;
use sp_consensus_aura::sr25519::AuthorityPair;
use sc_consensus_slots::{SimpleSlotWorker, BackoffAuthoringOnFinalizedHeadLagging};
use std::{task::Poll, time::{Instant, Duration}};
use sc_block_builder::BlockBuilderProvider;
use sp_runtime::traits::Header as _;
use substrate_test_runtime_client::{TestClient, runtime::{Header, H256}};
use sc_client_api::BlockchainEvents;
use sc_consensus_slots::{BackoffAuthoringOnFinalizedHeadLagging, SimpleSlotWorker};
use sc_keystore::LocalKeystore;
use sc_network::config::ProtocolConfig;
use sc_network_test::{Block as TestBlock, *};
use sp_application_crypto::key_types::AURA;
use sp_consensus::{
import_queue::BoxJustificationImport, AlwaysCanAuthor, DisableProofRecording,
NoNetwork as DummyOracle, Proposal, SlotData,
};
use sp_consensus_aura::sr25519::AuthorityPair;
use sp_inherents::InherentData;
use sp_keyring::sr25519::Keyring;
use sp_runtime::traits::{Block as BlockT, DigestFor, Header as _};
use sp_timestamp::InherentDataProvider as TimestampInherentDataProvider;
use std::{
task::Poll,
time::{Duration, Instant},
};
use substrate_test_runtime_client::{
runtime::{Header, H256},
TestClient,
};
type Error = sp_blockchain::Error;
@@ -576,19 +594,15 @@ mod tests {
type CreateProposer = futures::future::Ready<Result<DummyProposer, Error>>;
type Error = Error;
fn init(&mut self, parent_header: &<TestBlock as BlockT>::Header)
-> Self::CreateProposer
{
fn init(&mut self, parent_header: &<TestBlock as BlockT>::Header) -> Self::CreateProposer {
futures::future::ready(Ok(DummyProposer(parent_header.number + 1, self.0.clone())))
}
}
impl Proposer<TestBlock> for DummyProposer {
type Error = Error;
type Transaction = sc_client_api::TransactionFor<
substrate_test_runtime_client::Backend,
TestBlock
>;
type Transaction =
sc_client_api::TransactionFor<substrate_test_runtime_client::Backend, TestBlock>;
type Proposal = future::Ready<Result<Proposal<TestBlock, Self::Transaction, ()>, Error>>;
type ProofRecording = DisableProofRecording;
type Proof = ();
@@ -616,11 +630,13 @@ mod tests {
PeersFullClient,
AuthorityPair,
AlwaysCanAuthor,
Box<dyn CreateInherentDataProviders<
TestBlock,
(),
InherentDataProviders = (TimestampInherentDataProvider, InherentDataProvider)
>>
Box<
dyn CreateInherentDataProviders<
TestBlock,
(),
InherentDataProviders = (TimestampInherentDataProvider, InherentDataProvider),
>,
>,
>;
type AuraPeer = Peer<(), PeersClient>;
@@ -635,14 +651,15 @@ mod tests {
/// Create new test network with peers and given config.
fn from_config(_config: &ProtocolConfig) -> Self {
AuraTestNet {
peers: Vec::new(),
}
AuraTestNet { peers: Vec::new() }
}
fn make_verifier(&self, client: PeersClient, _cfg: &ProtocolConfig, _peer_data: &())
-> Self::Verifier
{
fn make_verifier(
&self,
client: PeersClient,
_cfg: &ProtocolConfig,
_peer_data: &(),
) -> Self::Verifier {
match client {
PeersClient::Full(client, _) => {
let slot_duration = slot_duration(&*client).expect("slot duration available");
@@ -668,7 +685,10 @@ mod tests {
}
}
fn make_block_import(&self, client: PeersClient) -> (
fn make_block_import(
&self,
client: PeersClient,
) -> (
BlockImportAdapter<Self::BlockImport>,
Option<BoxJustificationImport<Block>>,
Self::PeerData,
@@ -693,11 +713,7 @@ mod tests {
sp_tracing::try_init_simple();
let net = AuraTestNet::new(3);
let peers = &[
(0, Keyring::Alice),
(1, Keyring::Bob),
(2, Keyring::Charlie),
];
let peers = &[(0, Keyring::Alice), (1, Keyring::Bob), (2, Keyring::Charlie)];
let net = Arc::new(Mutex::new(net));
let mut import_notifications = Vec::new();
@@ -710,9 +726,9 @@ mod tests {
let client = peer.client().as_full().expect("full clients are created").clone();
let select_chain = peer.select_chain().expect("full client has a select chain");
let keystore_path = tempfile::tempdir().expect("Creates keystore path");
let keystore = Arc::new(LocalKeystore::open(keystore_path.path(), None)
.expect("Creates keystore."));
let keystore = Arc::new(
LocalKeystore::open(keystore_path.path(), None).expect("Creates keystore."),
);
SyncCryptoStore::sr25519_generate_new(&*keystore, AURA, Some(&key.to_seed()))
.expect("Creates authority key");
@@ -720,38 +736,46 @@ mod tests {
let environ = DummyFactory(client.clone());
import_notifications.push(
client.import_notification_stream()
.take_while(|n| future::ready(!(n.origin != BlockOrigin::Own && n.header.number() < &5)))
.for_each(move |_| future::ready(()))
client
.import_notification_stream()
.take_while(|n| {
future::ready(!(n.origin != BlockOrigin::Own && n.header.number() < &5))
})
.for_each(move |_| future::ready(())),
);
let slot_duration = slot_duration(&*client).expect("slot duration available");
aura_futures.push(start_aura::<AuthorityPair, _, _, _, _, _, _, _, _, _, _, _>(StartAuraParams {
slot_duration,
block_import: client.clone(),
select_chain,
client,
proposer_factory: environ,
sync_oracle: DummyOracle,
justification_sync_link: (),
create_inherent_data_providers: |_, _| async {
let timestamp = TimestampInherentDataProvider::from_system_time();
let slot = InherentDataProvider::from_timestamp_and_duration(
*timestamp,
Duration::from_secs(6),
);
aura_futures.push(
start_aura::<AuthorityPair, _, _, _, _, _, _, _, _, _, _, _>(StartAuraParams {
slot_duration,
block_import: client.clone(),
select_chain,
client,
proposer_factory: environ,
sync_oracle: DummyOracle,
justification_sync_link: (),
create_inherent_data_providers: |_, _| async {
let timestamp = TimestampInherentDataProvider::from_system_time();
let slot = InherentDataProvider::from_timestamp_and_duration(
*timestamp,
Duration::from_secs(6),
);
Ok((timestamp, slot))
},
force_authoring: false,
backoff_authoring_blocks: Some(BackoffAuthoringOnFinalizedHeadLagging::default()),
keystore,
can_author_with: sp_consensus::AlwaysCanAuthor,
block_proposal_slot_portion: SlotProportion::new(0.5),
max_block_proposal_slot_portion: None,
telemetry: None,
}).expect("Starts aura"));
Ok((timestamp, slot))
},
force_authoring: false,
backoff_authoring_blocks: Some(
BackoffAuthoringOnFinalizedHeadLagging::default(),
),
keystore,
can_author_with: sp_consensus::AlwaysCanAuthor,
block_proposal_slot_portion: SlotProportion::new(0.5),
max_block_proposal_slot_portion: None,
telemetry: None,
})
.expect("Starts aura"),
);
}
futures::executor::block_on(future::select(
@@ -759,10 +783,7 @@ mod tests {
net.lock().poll(cx);
Poll::<()>::Pending
}),
future::select(
future::join_all(aura_futures),
future::join_all(import_notifications)
)
future::select(future::join_all(aura_futures), future::join_all(import_notifications)),
));
}
@@ -771,11 +792,14 @@ mod tests {
let client = substrate_test_runtime_client::new();
assert_eq!(client.chain_info().best_number, 0);
assert_eq!(authorities(&client, &BlockId::Number(0)).unwrap(), vec![
Keyring::Alice.public().into(),
Keyring::Bob.public().into(),
Keyring::Charlie.public().into()
]);
assert_eq!(
authorities(&client, &BlockId::Number(0)).unwrap(),
vec![
Keyring::Alice.public().into(),
Keyring::Bob.public().into(),
Keyring::Charlie.public().into()
]
);
}
#[test]
@@ -785,12 +809,11 @@ mod tests {
let mut authorities = vec![
Keyring::Alice.public().into(),
Keyring::Bob.public().into(),
Keyring::Charlie.public().into()
Keyring::Charlie.public().into(),
];
let keystore_path = tempfile::tempdir().expect("Creates keystore path");
let keystore = LocalKeystore::open(keystore_path.path(), None)
.expect("Creates keystore.");
let keystore = LocalKeystore::open(keystore_path.path(), None).expect("Creates keystore.");
let public = SyncCryptoStore::sr25519_generate_new(&keystore, AuthorityPair::ID, None)
.expect("Key should be created");
authorities.push(public.into());
@@ -822,7 +845,7 @@ mod tests {
H256::from_low_u64_be(0),
H256::from_low_u64_be(0),
Default::default(),
Default::default()
Default::default(),
);
assert!(worker.claim_slot(&head, 0.into(), &authorities).is_none());
assert!(worker.claim_slot(&head, 1.into(), &authorities).is_none());
@@ -839,12 +862,13 @@ mod tests {
let net = AuraTestNet::new(4);
let keystore_path = tempfile::tempdir().expect("Creates keystore path");
let keystore = LocalKeystore::open(keystore_path.path(), None)
.expect("Creates keystore.");
let keystore = LocalKeystore::open(keystore_path.path(), None).expect("Creates keystore.");
SyncCryptoStore::sr25519_generate_new(
&keystore,
AuthorityPair::ID, Some(&Keyring::Alice.to_seed()),
).expect("Key should be created");
AuthorityPair::ID,
Some(&Keyring::Alice.to_seed()),
)
.expect("Key should be created");
let net = Arc::new(Mutex::new(net));
@@ -870,17 +894,16 @@ mod tests {
let head = client.header(&BlockId::Number(0)).unwrap().unwrap();
let res = futures::executor::block_on(worker.on_slot(
SlotInfo {
slot: 0.into(),
timestamp: 0.into(),
ends_at: Instant::now() + Duration::from_secs(100),
inherent_data: InherentData::new(),
duration: Duration::from_millis(1000),
chain_head: head,
block_size_limit: None,
}
)).unwrap();
let res = futures::executor::block_on(worker.on_slot(SlotInfo {
slot: 0.into(),
timestamp: 0.into(),
ends_at: Instant::now() + Duration::from_secs(100),
inherent_data: InherentData::new(),
duration: Duration::from_millis(1000),
chain_head: head,
block_size_limit: None,
}))
.unwrap();
// The returned block should be imported and we should be able to get its header by now.
assert!(client.header(&BlockId::Hash(res.block.hash())).unwrap().is_some());
+43 -75
View File
@@ -18,30 +18,21 @@
//! RPC api for babe.
use sc_consensus_babe::{Epoch, authorship, Config};
use futures::{FutureExt as _, TryFutureExt as _};
use jsonrpc_core::{
Error as RpcError,
futures::future as rpc_future,
};
use jsonrpc_core::{futures::future as rpc_future, Error as RpcError};
use jsonrpc_derive::rpc;
use sc_consensus_babe::{authorship, Config, Epoch};
use sc_consensus_epochs::{descendent_query, Epoch as EpochT, SharedEpochChanges};
use sp_consensus_babe::{
AuthorityId,
BabeApi as BabeRuntimeApi,
digests::PreDigest,
};
use serde::{Deserialize, Serialize};
use sp_core::{
crypto::Public,
};
use sp_application_crypto::AppKey;
use sp_keystore::{SyncCryptoStorePtr, SyncCryptoStore};
use sc_rpc_api::DenyUnsafe;
use sp_api::{ProvideRuntimeApi, BlockId};
use serde::{Deserialize, Serialize};
use sp_api::{BlockId, ProvideRuntimeApi};
use sp_application_crypto::AppKey;
use sp_blockchain::{Error as BlockChainError, HeaderBackend, HeaderMetadata};
use sp_consensus::{Error as ConsensusError, SelectChain};
use sp_consensus_babe::{digests::PreDigest, AuthorityId, BabeApi as BabeRuntimeApi};
use sp_core::crypto::Public;
use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr};
use sp_runtime::traits::{Block as BlockT, Header as _};
use sp_consensus::{SelectChain, Error as ConsensusError};
use sp_blockchain::{HeaderBackend, HeaderMetadata, Error as BlockChainError};
use std::{collections::HashMap, sync::Arc};
type FutureResult<T> = Box<dyn rpc_future::Future<Item = T, Error = RpcError> + Send>;
@@ -81,14 +72,7 @@ impl<B: BlockT, C, SC> BabeRpcHandler<B, C, SC> {
select_chain: SC,
deny_unsafe: DenyUnsafe,
) -> Self {
Self {
client,
shared_epoch_changes,
keystore,
babe_config,
select_chain,
deny_unsafe,
}
Self { client, shared_epoch_changes, keystore, babe_config, select_chain, deny_unsafe }
}
}
@@ -104,16 +88,10 @@ where
{
fn epoch_authorship(&self) -> FutureResult<HashMap<AuthorityId, EpochAuthorship>> {
if let Err(err) = self.deny_unsafe.check_if_safe() {
return Box::new(rpc_future::err(err.into()));
return Box::new(rpc_future::err(err.into()))
}
let (
babe_config,
keystore,
shared_epoch,
client,
select_chain,
) = (
let (babe_config, keystore, shared_epoch, client, select_chain) = (
self.babe_config.clone(),
self.keystore.clone(),
self.shared_epoch_changes.clone(),
@@ -126,14 +104,9 @@ where
.runtime_api()
.current_epoch_start(&BlockId::Hash(header.hash()))
.map_err(|err| Error::StringError(format!("{:?}", err)))?;
let epoch = epoch_data(
&shared_epoch,
&client,
&babe_config,
*epoch_start,
&select_chain,
)
.await?;
let epoch =
epoch_data(&shared_epoch, &client, &babe_config, *epoch_start, &select_chain)
.await?;
let (epoch_start, epoch_end) = (epoch.start_slot(), epoch.end_slot());
let mut claims: HashMap<AuthorityId, EpochAuthorship> = HashMap::new();
@@ -163,10 +136,10 @@ where
match claim {
PreDigest::Primary { .. } => {
claims.entry(key).or_default().primary.push(slot);
}
},
PreDigest::SecondaryPlain { .. } => {
claims.entry(key).or_default().secondary.push(slot);
}
},
PreDigest::SecondaryVRF { .. } => {
claims.entry(key).or_default().secondary_vrf.push(slot.into());
},
@@ -199,7 +172,7 @@ pub enum Error {
/// Consensus error
Consensus(ConsensusError),
/// Errors that can be formatted as a String
StringError(String)
StringError(String),
}
impl From<Error> for jsonrpc_core::Error {
@@ -226,13 +199,15 @@ where
SC: SelectChain<B>,
{
let parent = select_chain.best_chain().await?;
epoch_changes.shared_data().epoch_data_for_child_of(
descendent_query(&**client),
&parent.hash(),
parent.number().clone(),
slot.into(),
|slot| Epoch::genesis(&babe_config, slot),
)
epoch_changes
.shared_data()
.epoch_data_for_child_of(
descendent_query(&**client),
&parent.hash(),
parent.number().clone(),
slot.into(),
|slot| Epoch::genesis(&babe_config, slot),
)
.map_err(|e| Error::Consensus(ConsensusError::ChainLookup(format!("{:?}", e))))?
.ok_or(Error::Consensus(ConsensusError::InvalidAuthoritiesSet))
}
@@ -240,31 +215,27 @@ where
#[cfg(test)]
mod tests {
use super::*;
use substrate_test_runtime_client::{
runtime::Block,
Backend,
DefaultTestClientBuilderExt,
TestClient,
TestClientBuilderExt,
TestClientBuilder,
};
use sp_application_crypto::AppPair;
use sp_keyring::Sr25519Keyring;
use sp_core::{crypto::key_types::BABE};
use sp_keystore::{SyncCryptoStorePtr, SyncCryptoStore};
use sc_keystore::LocalKeystore;
use sp_application_crypto::AppPair;
use sp_core::crypto::key_types::BABE;
use sp_keyring::Sr25519Keyring;
use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr};
use substrate_test_runtime_client::{
runtime::Block, Backend, DefaultTestClientBuilderExt, TestClient, TestClientBuilder,
TestClientBuilderExt,
};
use std::sync::Arc;
use sc_consensus_babe::{Config, block_import, AuthorityPair};
use jsonrpc_core::IoHandler;
use sc_consensus_babe::{block_import, AuthorityPair, Config};
use std::sync::Arc;
/// creates keystore backed by a temp file
fn create_temp_keystore<P: AppPair>(
authority: Sr25519Keyring,
) -> (SyncCryptoStorePtr, tempfile::TempDir) {
let keystore_path = tempfile::tempdir().expect("Creates keystore path");
let keystore = Arc::new(LocalKeystore::open(keystore_path.path(), None)
.expect("Creates keystore"));
let keystore =
Arc::new(LocalKeystore::open(keystore_path.path(), None).expect("Creates keystore"));
SyncCryptoStore::sr25519_generate_new(&*keystore, BABE, Some(&authority.to_seed()))
.expect("Creates authority key");
@@ -272,17 +243,14 @@ mod tests {
}
fn test_babe_rpc_handler(
deny_unsafe: DenyUnsafe
deny_unsafe: DenyUnsafe,
) -> BabeRpcHandler<Block, TestClient, sc_consensus::LongestChain<Backend, Block>> {
let builder = TestClientBuilder::new();
let (client, longest_chain) = builder.build_with_longest_chain();
let client = Arc::new(client);
let config = Config::get_or_compute(&*client).expect("config available");
let (_, link) = block_import(
config.clone(),
client.clone(),
client.clone(),
).expect("can initialize block-import");
let (_, link) = block_import(config.clone(), client.clone(), client.clone())
.expect("can initialize block-import");
let epoch_changes = link.epoch_changes().clone();
let keystore = create_temp_keystore::<AuthorityPair>(Sr25519Keyring::Alice).0;
@@ -18,23 +18,17 @@
//! BABE authority selection and slot claiming.
use super::Epoch;
use codec::Encode;
use schnorrkel::{keys::PublicKey, vrf::VRFInOut};
use sp_application_crypto::AppKey;
use sp_consensus_babe::{
BABE_VRF_PREFIX, AuthorityId, BabeAuthorityWeight, make_transcript, make_transcript_data,
Slot,
};
use sp_consensus_babe::digests::{
PreDigest, PrimaryPreDigest, SecondaryPlainPreDigest, SecondaryVRFPreDigest,
digests::{PreDigest, PrimaryPreDigest, SecondaryPlainPreDigest, SecondaryVRFPreDigest},
make_transcript, make_transcript_data, AuthorityId, BabeAuthorityWeight, Slot, BABE_VRF_PREFIX,
};
use sp_consensus_vrf::schnorrkel::{VRFOutput, VRFProof};
use sp_core::{U256, blake2_256, crypto::Public};
use sp_keystore::{SyncCryptoStorePtr, SyncCryptoStore};
use codec::Encode;
use schnorrkel::{
keys::PublicKey,
vrf::VRFInOut,
};
use super::Epoch;
use sp_core::{blake2_256, crypto::Public, U256};
use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr};
/// Calculates the primary selection threshold for a given authority, taking
/// into account `c` (`1 - c` represents the probability of a slot being empty).
@@ -49,8 +43,7 @@ pub(super) fn calculate_primary_threshold(
let c = c.0 as f64 / c.1 as f64;
let theta =
authorities[authority_index].1 as f64 /
let theta = authorities[authority_index].1 as f64 /
authorities.iter().map(|(_, weight)| weight).sum::<u64>() as f64;
assert!(theta > 0.0, "authority with weight 0.");
@@ -74,14 +67,14 @@ pub(super) fn calculate_primary_threshold(
"returns None when the given value is negative; \
p is defined as `1 - n` where n is defined in (0, 1]; \
p must be a value in [0, 1); \
qed."
qed.",
);
let denom = p.denom().to_biguint().expect(
"returns None when the given value is negative; \
p is defined as `1 - n` where n is defined in (0, 1]; \
p must be a value in [0, 1); \
qed."
qed.",
);
((BigUint::one() << 128) * numer / denom).to_u128().expect(
@@ -108,7 +101,7 @@ pub(super) fn secondary_slot_author(
randomness: [u8; 32],
) -> Option<&AuthorityId> {
if authorities.is_empty() {
return None;
return None
}
let rand = U256::from((randomness, slot).using_encoded(blake2_256));
@@ -116,9 +109,10 @@ pub(super) fn secondary_slot_author(
let authorities_len = U256::from(authorities.len());
let idx = rand % authorities_len;
let expected_author = authorities.get(idx.as_u32() as usize)
.expect("authorities not empty; index constrained to list length; \
this is a valid index; qed");
let expected_author = authorities.get(idx.as_u32() as usize).expect(
"authorities not empty; index constrained to list length; \
this is a valid index; qed",
);
Some(&expected_author.0)
}
@@ -136,23 +130,15 @@ fn claim_secondary_slot(
let Epoch { authorities, randomness, epoch_index, .. } = epoch;
if authorities.is_empty() {
return None;
return None
}
let expected_author = secondary_slot_author(
slot,
authorities,
*randomness,
)?;
let expected_author = secondary_slot_author(slot, authorities, *randomness)?;
for (authority_id, authority_index) in keys {
if authority_id == expected_author {
let pre_digest = if author_secondary_vrf {
let transcript_data = make_transcript_data(
randomness,
slot,
*epoch_index,
);
let transcript_data = make_transcript_data(randomness, slot, *epoch_index);
let result = SyncCryptoStore::sr25519_vrf_sign(
&**keystore,
AuthorityId::ID,
@@ -169,7 +155,10 @@ fn claim_secondary_slot(
} else {
None
}
} else if SyncCryptoStore::has_keys(&**keystore, &[(authority_id.to_raw_vec(), AuthorityId::ID)]) {
} else if SyncCryptoStore::has_keys(
&**keystore,
&[(authority_id.to_raw_vec(), AuthorityId::ID)],
) {
Some(PreDigest::SecondaryPlain(SecondaryPlainPreDigest {
slot,
authority_index: *authority_index as u32,
@@ -179,7 +168,7 @@ fn claim_secondary_slot(
};
if let Some(pre_digest) = pre_digest {
return Some((pre_digest, authority_id.clone()));
return Some((pre_digest, authority_id.clone()))
}
}
}
@@ -196,7 +185,9 @@ pub fn claim_slot(
epoch: &Epoch,
keystore: &SyncCryptoStorePtr,
) -> Option<(PreDigest, AuthorityId)> {
let authorities = epoch.authorities.iter()
let authorities = epoch
.authorities
.iter()
.enumerate()
.map(|(index, a)| (a.0.clone(), index))
.collect::<Vec<_>>();
@@ -211,22 +202,21 @@ pub fn claim_slot_using_keys(
keystore: &SyncCryptoStorePtr,
keys: &[(AuthorityId, usize)],
) -> Option<(PreDigest, AuthorityId)> {
claim_primary_slot(slot, epoch, epoch.config.c, keystore, &keys)
.or_else(|| {
if epoch.config.allowed_slots.is_secondary_plain_slots_allowed() ||
epoch.config.allowed_slots.is_secondary_vrf_slots_allowed()
{
claim_secondary_slot(
slot,
&epoch,
keys,
&keystore,
epoch.config.allowed_slots.is_secondary_vrf_slots_allowed(),
)
} else {
None
}
})
claim_primary_slot(slot, epoch, epoch.config.c, keystore, &keys).or_else(|| {
if epoch.config.allowed_slots.is_secondary_plain_slots_allowed() ||
epoch.config.allowed_slots.is_secondary_vrf_slots_allowed()
{
claim_secondary_slot(
slot,
&epoch,
keys,
&keystore,
epoch.config.allowed_slots.is_secondary_vrf_slots_allowed(),
)
} else {
None
}
})
}
/// Claim a primary slot if it is our turn. Returns `None` if it is not our turn.
@@ -243,16 +233,8 @@ fn claim_primary_slot(
let Epoch { authorities, randomness, epoch_index, .. } = epoch;
for (authority_id, authority_index) in keys {
let transcript = make_transcript(
randomness,
slot,
*epoch_index
);
let transcript_data = make_transcript_data(
randomness,
slot,
*epoch_index
);
let transcript = make_transcript(randomness, slot, *epoch_index);
let transcript_data = make_transcript_data(randomness, slot, *epoch_index);
// Compute the threshold we will use.
//
// We already checked that authorities contains `key.public()`, so it can't
@@ -279,7 +261,7 @@ fn claim_primary_slot(
authority_index: *authority_index as u32,
});
return Some((pre_digest, authority_id.clone()));
return Some((pre_digest, authority_id.clone()))
}
}
}
@@ -290,10 +272,10 @@ fn claim_primary_slot(
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
use sp_core::{sr25519::Pair, crypto::Pair as _};
use sp_consensus_babe::{AuthorityId, BabeEpochConfiguration, AllowedSlots};
use sc_keystore::LocalKeystore;
use sp_consensus_babe::{AllowedSlots, AuthorityId, BabeEpochConfiguration};
use sp_core::{crypto::Pair as _, sr25519::Pair};
use std::sync::Arc;
#[test]
fn claim_secondary_plain_slot_works() {
@@ -302,7 +284,8 @@ mod tests {
&*keystore,
AuthorityId::ID,
Some(sp_core::crypto::DEV_PHRASE),
).unwrap();
)
.unwrap();
let authorities = vec![
(AuthorityId::from(Pair::generate().0.public()), 5),
@@ -18,15 +18,15 @@
//! Schema for BABE epoch changes in the aux-db.
use log::info;
use codec::{Decode, Encode};
use log::info;
use crate::{migration::EpochV0, Epoch};
use sc_client_api::backend::AuxStore;
use sp_blockchain::{Result as ClientResult, Error as ClientError};
use sp_runtime::traits::Block as BlockT;
use sc_consensus_epochs::{migration::EpochChangesForV0, EpochChangesFor, SharedEpochChanges};
use sp_blockchain::{Error as ClientError, Result as ClientResult};
use sp_consensus_babe::{BabeBlockWeight, BabeGenesisConfiguration};
use sc_consensus_epochs::{EpochChangesFor, SharedEpochChanges, migration::EpochChangesForV0};
use crate::{Epoch, migration::EpochV0};
use sp_runtime::traits::Block as BlockT;
const BABE_EPOCH_CHANGES_VERSION: &[u8] = b"babe_epoch_changes_version";
const BABE_EPOCH_CHANGES_KEY: &[u8] = b"babe_epoch_changes";
@@ -38,16 +38,16 @@ pub fn block_weight_key<H: Encode>(block_hash: H) -> Vec<u8> {
}
fn load_decode<B, T>(backend: &B, key: &[u8]) -> ClientResult<Option<T>>
where
B: AuxStore,
T: Decode,
where
B: AuxStore,
T: Decode,
{
let corrupt = |e: codec::Error| {
ClientError::Backend(format!("BABE DB is corrupted. Decode error: {}", e))
};
match backend.get_aux(key)? {
None => Ok(None),
Some(t) => T::decode(&mut &t[..]).map(Some).map_err(corrupt)
Some(t) => T::decode(&mut &t[..]).map(Some).map_err(corrupt),
}
}
@@ -59,32 +59,26 @@ pub fn load_epoch_changes<Block: BlockT, B: AuxStore>(
let version = load_decode::<_, u32>(backend, BABE_EPOCH_CHANGES_VERSION)?;
let maybe_epoch_changes = match version {
None => load_decode::<_, EpochChangesForV0<Block, EpochV0>>(
backend,
BABE_EPOCH_CHANGES_KEY,
)?.map(|v0| v0.migrate().map(|_, _, epoch| epoch.migrate(config))),
Some(1) => load_decode::<_, EpochChangesFor<Block, EpochV0>>(
backend,
BABE_EPOCH_CHANGES_KEY,
)?.map(|v1| v1.map(|_, _, epoch| epoch.migrate(config))),
Some(BABE_EPOCH_CHANGES_CURRENT_VERSION) => load_decode::<_, EpochChangesFor<Block, Epoch>>(
backend,
BABE_EPOCH_CHANGES_KEY,
)?,
Some(other) => {
return Err(ClientError::Backend(
format!("Unsupported BABE DB version: {:?}", other)
))
},
None =>
load_decode::<_, EpochChangesForV0<Block, EpochV0>>(backend, BABE_EPOCH_CHANGES_KEY)?
.map(|v0| v0.migrate().map(|_, _, epoch| epoch.migrate(config))),
Some(1) =>
load_decode::<_, EpochChangesFor<Block, EpochV0>>(backend, BABE_EPOCH_CHANGES_KEY)?
.map(|v1| v1.map(|_, _, epoch| epoch.migrate(config))),
Some(BABE_EPOCH_CHANGES_CURRENT_VERSION) =>
load_decode::<_, EpochChangesFor<Block, Epoch>>(backend, BABE_EPOCH_CHANGES_KEY)?,
Some(other) =>
return Err(ClientError::Backend(format!("Unsupported BABE DB version: {:?}", other))),
};
let epoch_changes = SharedEpochChanges::<Block, Epoch>::new(maybe_epoch_changes.unwrap_or_else(|| {
info!(
target: "babe",
"👶 Creating empty BABE epoch changes on what appears to be first startup.",
);
EpochChangesFor::<Block, Epoch>::default()
}));
let epoch_changes =
SharedEpochChanges::<Block, Epoch>::new(maybe_epoch_changes.unwrap_or_else(|| {
info!(
target: "babe",
"👶 Creating empty BABE epoch changes on what appears to be first startup.",
);
EpochChangesFor::<Block, Epoch>::default()
}));
// rebalance the tree after deserialization. this isn't strictly necessary
// since the tree is now rebalanced on every update operation. but since the
@@ -99,15 +93,16 @@ pub fn load_epoch_changes<Block: BlockT, B: AuxStore>(
pub(crate) fn write_epoch_changes<Block: BlockT, F, R>(
epoch_changes: &EpochChangesFor<Block, Epoch>,
write_aux: F,
) -> R where
) -> R
where
F: FnOnce(&[(&'static [u8], &[u8])]) -> R,
{
BABE_EPOCH_CHANGES_CURRENT_VERSION.using_encoded(|version| {
let encoded_epoch_changes = epoch_changes.encode();
write_aux(
&[(BABE_EPOCH_CHANGES_KEY, encoded_epoch_changes.as_slice()),
(BABE_EPOCH_CHANGES_VERSION, version)],
)
write_aux(&[
(BABE_EPOCH_CHANGES_KEY, encoded_epoch_changes.as_slice()),
(BABE_EPOCH_CHANGES_VERSION, version),
])
})
}
@@ -116,15 +111,12 @@ pub(crate) fn write_block_weight<H: Encode, F, R>(
block_hash: H,
block_weight: BabeBlockWeight,
write_aux: F,
) -> R where
) -> R
where
F: FnOnce(&[(Vec<u8>, &[u8])]) -> R,
{
let key = block_weight_key(block_hash);
block_weight.using_encoded(|s|
write_aux(
&[(key, s)],
)
)
block_weight.using_encoded(|s| write_aux(&[(key, s)]))
}
/// Load the cumulative chain-weight associated with a block.
@@ -140,13 +132,13 @@ mod test {
use super::*;
use crate::migration::EpochV0;
use fork_tree::ForkTree;
use substrate_test_runtime_client;
use sc_consensus_epochs::{EpochHeader, PersistedEpoch, PersistedEpochHeader};
use sc_network_test::Block as TestBlock;
use sp_consensus::Error as ConsensusError;
use sp_consensus_babe::{AllowedSlots, BabeGenesisConfiguration};
use sp_core::H256;
use sp_runtime::traits::NumberFor;
use sp_consensus_babe::{AllowedSlots, BabeGenesisConfiguration};
use sc_consensus_epochs::{PersistedEpoch, PersistedEpochHeader, EpochHeader};
use sp_consensus::Error as ConsensusError;
use sc_network_test::Block as TestBlock;
use substrate_test_runtime_client;
#[test]
fn load_decode_from_v0_epoch_changes() {
@@ -159,26 +151,30 @@ mod test {
};
let client = substrate_test_runtime_client::new();
let mut v0_tree = ForkTree::<H256, NumberFor<TestBlock>, _>::new();
v0_tree.import::<_, ConsensusError>(
Default::default(),
Default::default(),
PersistedEpoch::Regular(epoch),
&|_, _| Ok(false), // Test is single item only so this can be set to false.
).unwrap();
v0_tree
.import::<_, ConsensusError>(
Default::default(),
Default::default(),
PersistedEpoch::Regular(epoch),
&|_, _| Ok(false), // Test is single item only so this can be set to false.
)
.unwrap();
client.insert_aux(
&[(BABE_EPOCH_CHANGES_KEY,
&EpochChangesForV0::<TestBlock, EpochV0>::from_raw(v0_tree).encode()[..])],
&[],
).unwrap();
client
.insert_aux(
&[(
BABE_EPOCH_CHANGES_KEY,
&EpochChangesForV0::<TestBlock, EpochV0>::from_raw(v0_tree).encode()[..],
)],
&[],
)
.unwrap();
assert_eq!(
load_decode::<_, u32>(&client, BABE_EPOCH_CHANGES_VERSION).unwrap(),
None,
);
assert_eq!(load_decode::<_, u32>(&client, BABE_EPOCH_CHANGES_VERSION).unwrap(), None,);
let epoch_changes = load_epoch_changes::<TestBlock, _>(
&client, &BabeGenesisConfiguration {
&client,
&BabeGenesisConfiguration {
slot_duration: 10,
epoch_length: 4,
c: (3, 10),
@@ -186,10 +182,12 @@ mod test {
randomness: Default::default(),
allowed_slots: AllowedSlots::PrimaryAndSecondaryPlainSlots,
},
).unwrap();
)
.unwrap();
assert!(
epoch_changes.shared_data()
epoch_changes
.shared_data()
.tree()
.iter()
.map(|(_, _, epoch)| epoch.clone())
@@ -200,16 +198,10 @@ mod test {
})],
); // PersistedEpochHeader does not implement Debug, so we use assert! directly.
write_epoch_changes::<TestBlock, _, _>(
&epoch_changes.shared_data(),
|values| {
client.insert_aux(values, &[]).unwrap();
},
);
write_epoch_changes::<TestBlock, _, _>(&epoch_changes.shared_data(), |values| {
client.insert_aux(values, &[]).unwrap();
});
assert_eq!(
load_decode::<_, u32>(&client, BABE_EPOCH_CHANGES_VERSION).unwrap(),
Some(2),
);
assert_eq!(load_decode::<_, u32>(&client, BABE_EPOCH_CHANGES_VERSION).unwrap(), Some(2),);
}
}
File diff suppressed because it is too large Load Diff
@@ -16,12 +16,12 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use codec::{Encode, Decode};
use sc_consensus_epochs::Epoch as EpochT;
use crate::{
Epoch, AuthorityId, BabeAuthorityWeight, BabeGenesisConfiguration,
BabeEpochConfiguration, VRF_OUTPUT_LENGTH, NextEpochDescriptor,
AuthorityId, BabeAuthorityWeight, BabeEpochConfiguration, BabeGenesisConfiguration, Epoch,
NextEpochDescriptor, VRF_OUTPUT_LENGTH,
};
use codec::{Decode, Encode};
use sc_consensus_epochs::Epoch as EpochT;
use sp_consensus_slots::Slot;
/// BABE epoch information, version 0.
@@ -43,10 +43,7 @@ impl EpochT for EpochV0 {
type NextEpochDescriptor = NextEpochDescriptor;
type Slot = Slot;
fn increment(
&self,
descriptor: NextEpochDescriptor
) -> EpochV0 {
fn increment(&self, descriptor: NextEpochDescriptor) -> EpochV0 {
EpochV0 {
epoch_index: self.epoch_index + 1,
start_slot: self.start_slot + self.duration,
@@ -74,10 +71,7 @@ impl EpochV0 {
duration: self.duration,
authorities: self.authorities,
randomness: self.randomness,
config: BabeEpochConfiguration {
c: config.c,
allowed_slots: config.allowed_slots,
},
config: BabeEpochConfiguration { c: config.c, allowed_slots: config.allowed_slots },
}
}
}
+207 -204
View File
@@ -23,35 +23,33 @@
#![allow(deprecated)]
use super::*;
use authorship::claim_slot;
use sp_core::crypto::Pair;
use sp_keystore::{
SyncCryptoStore,
vrf::make_transcript as transcript_from_data,
use futures::executor::block_on;
use log::debug;
use rand::RngCore;
use rand_chacha::{rand_core::SeedableRng, ChaChaRng};
use sc_block_builder::{BlockBuilder, BlockBuilderProvider};
use sc_client_api::{backend::TransactionFor, BlockchainEvents};
use sc_consensus_slots::BackoffAuthoringOnFinalizedHeadLagging;
use sc_keystore::LocalKeystore;
use sc_network::config::ProtocolConfig;
use sc_network_test::{Block as TestBlock, *};
use sp_application_crypto::key_types::BABE;
use sp_consensus::{
import_queue::{BoxBlockImport, BoxJustificationImport},
AlwaysCanAuthor, DisableProofRecording, NoNetwork as DummyOracle, Proposal,
};
use sp_consensus_babe::{
AuthorityPair, Slot, AllowedSlots, make_transcript, make_transcript_data,
inherents::InherentDataProvider,
inherents::InherentDataProvider, make_transcript, make_transcript_data, AllowedSlots,
AuthorityPair, Slot,
};
use sc_consensus_slots::BackoffAuthoringOnFinalizedHeadLagging;
use sc_block_builder::{BlockBuilder, BlockBuilderProvider};
use sp_consensus::{
NoNetwork as DummyOracle, Proposal, DisableProofRecording, AlwaysCanAuthor,
import_queue::{BoxBlockImport, BoxJustificationImport},
use sp_core::crypto::Pair;
use sp_keystore::{vrf::make_transcript as transcript_from_data, SyncCryptoStore};
use sp_runtime::{
generic::DigestItem,
traits::{Block as BlockT, DigestFor},
};
use sc_network_test::{Block as TestBlock, *};
use sc_network::config::ProtocolConfig;
use sp_runtime::{generic::DigestItem, traits::{Block as BlockT, DigestFor}};
use sc_client_api::{BlockchainEvents, backend::TransactionFor};
use log::debug;
use std::{time::Duration, cell::RefCell, task::Poll};
use rand::RngCore;
use rand_chacha::{
rand_core::SeedableRng, ChaChaRng,
};
use sc_keystore::LocalKeystore;
use sp_application_crypto::key_types::BABE;
use futures::executor::block_on;
use sp_timestamp::InherentDataProvider as TimestampInherentDataProvider;
use std::{cell::RefCell, task::Poll, time::Duration};
type Item = DigestItem<Hash>;
@@ -95,10 +93,7 @@ impl Environment<TestBlock> for DummyFactory {
type Proposer = DummyProposer;
type Error = Error;
fn init(&mut self, parent_header: &<TestBlock as BlockT>::Header)
-> Self::CreateProposer
{
fn init(&mut self, parent_header: &<TestBlock as BlockT>::Header) -> Self::CreateProposer {
let parent_slot = crate::find_pre_digest::<TestBlock>(parent_header)
.expect("parent header has a pre-digest")
.slot();
@@ -113,23 +108,24 @@ impl Environment<TestBlock> for DummyFactory {
}
impl DummyProposer {
fn propose_with(&mut self, pre_digests: DigestFor<TestBlock>)
-> future::Ready<
Result<
Proposal<
TestBlock,
sc_client_api::TransactionFor<substrate_test_runtime_client::Backend, TestBlock>,
()
>,
Error
>
>
{
let block_builder = self.factory.client.new_block_at(
&BlockId::Hash(self.parent_hash),
pre_digests,
false,
).unwrap();
fn propose_with(
&mut self,
pre_digests: DigestFor<TestBlock>,
) -> future::Ready<
Result<
Proposal<
TestBlock,
sc_client_api::TransactionFor<substrate_test_runtime_client::Backend, TestBlock>,
(),
>,
Error,
>,
> {
let block_builder = self
.factory
.client
.new_block_at(&BlockId::Hash(self.parent_hash), pre_digests, false)
.unwrap();
let mut block = match block_builder.build().map_err(|e| e.into()) {
Ok(b) => b.block,
@@ -143,13 +139,14 @@ impl DummyProposer {
// figure out if we should add a consensus digest, since the test runtime
// doesn't.
let epoch_changes = self.factory.epoch_changes.shared_data();
let epoch = epoch_changes.epoch_data_for_child_of(
descendent_query(&*self.factory.client),
&self.parent_hash,
self.parent_number,
this_slot,
|slot| Epoch::genesis(&self.factory.config, slot),
)
let epoch = epoch_changes
.epoch_data_for_child_of(
descendent_query(&*self.factory.client),
&self.parent_hash,
self.parent_number,
this_slot,
|slot| Epoch::genesis(&self.factory.config, slot),
)
.expect("client has data to find epoch")
.expect("can compute epoch for baked block");
@@ -162,7 +159,8 @@ impl DummyProposer {
let digest_data = ConsensusLog::NextEpochData(NextEpochDescriptor {
authorities: epoch.authorities.clone(),
randomness: epoch.randomness.clone(),
}).encode();
})
.encode();
let digest = DigestItem::Consensus(BABE_ENGINE_ID, digest_data);
block.header.digest_mut().push(digest)
}
@@ -176,7 +174,8 @@ impl DummyProposer {
impl Proposer<TestBlock> for DummyProposer {
type Error = Error;
type Transaction = sc_client_api::TransactionFor<substrate_test_runtime_client::Backend, TestBlock>;
type Transaction =
sc_client_api::TransactionFor<substrate_test_runtime_client::Backend, TestBlock>;
type Proposal = future::Ready<Result<Proposal<TestBlock, Self::Transaction, ()>, Error>>;
type ProofRecording = DisableProofRecording;
type Proof = ();
@@ -201,9 +200,9 @@ pub struct PanickingBlockImport<B>(B);
#[async_trait::async_trait]
impl<B: BlockImport<TestBlock>> BlockImport<TestBlock> for PanickingBlockImport<B>
where
B::Transaction: Send,
B: Send,
where
B::Transaction: Send,
B: Send,
{
type Error = B::Error;
type Transaction = B::Transaction;
@@ -233,10 +232,8 @@ pub struct BabeTestNet {
type TestHeader = <TestBlock as BlockT>::Header;
type TestExtrinsic = <TestBlock as BlockT>::Extrinsic;
type TestSelectChain = substrate_test_runtime_client::LongestChain<
substrate_test_runtime_client::Backend,
TestBlock,
>;
type TestSelectChain =
substrate_test_runtime_client::LongestChain<substrate_test_runtime_client::Backend, TestBlock>;
pub struct TestVerifier {
inner: BabeVerifier<
@@ -244,11 +241,13 @@ pub struct TestVerifier {
PeersFullClient,
TestSelectChain,
AlwaysCanAuthor,
Box<dyn CreateInherentDataProviders<
TestBlock,
(),
InherentDataProviders = (TimestampInherentDataProvider, InherentDataProvider)
>>
Box<
dyn CreateInherentDataProviders<
TestBlock,
(),
InherentDataProviders = (TimestampInherentDataProvider, InherentDataProvider),
>,
>,
>,
mutator: Mutator,
}
@@ -274,7 +273,12 @@ impl Verifier<TestBlock> for TestVerifier {
pub struct PeerData {
link: BabeLink<TestBlock>,
block_import: Mutex<
Option<BoxBlockImport<TestBlock, TransactionFor<substrate_test_runtime_client::Backend, TestBlock>>>
Option<
BoxBlockImport<
TestBlock,
TransactionFor<substrate_test_runtime_client::Backend, TestBlock>,
>,
>,
>,
}
@@ -286,32 +290,27 @@ impl TestNetFactory for BabeTestNet {
/// Create new test network with peers and given config.
fn from_config(_config: &ProtocolConfig) -> Self {
debug!(target: "babe", "Creating test network from config");
BabeTestNet {
peers: Vec::new(),
}
BabeTestNet { peers: Vec::new() }
}
fn make_block_import(&self, client: PeersClient)
-> (
BlockImportAdapter<Self::BlockImport>,
Option<BoxJustificationImport<Block>>,
Option<PeerData>,
)
{
fn make_block_import(
&self,
client: PeersClient,
) -> (
BlockImportAdapter<Self::BlockImport>,
Option<BoxJustificationImport<Block>>,
Option<PeerData>,
) {
let client = client.as_full().expect("only full clients are tested");
let config = Config::get_or_compute(&*client).expect("config available");
let (block_import, link) = crate::block_import(
config,
client.clone(),
client.clone(),
).expect("can initialize block-import");
let (block_import, link) = crate::block_import(config, client.clone(), client.clone())
.expect("can initialize block-import");
let block_import = PanickingBlockImport(block_import);
let data_block_import = Mutex::new(
Some(Box::new(block_import.clone()) as BoxBlockImport<_, _>)
);
let data_block_import =
Mutex::new(Some(Box::new(block_import.clone()) as BoxBlockImport<_, _>));
(
BlockImportAdapter::new(block_import),
None,
@@ -324,16 +323,16 @@ impl TestNetFactory for BabeTestNet {
client: PeersClient,
_cfg: &ProtocolConfig,
maybe_link: &Option<PeerData>,
)
-> Self::Verifier
{
) -> Self::Verifier {
use substrate_test_runtime_client::DefaultTestClientBuilderExt;
let client = client.as_full().expect("only full clients are used in test");
trace!(target: "babe", "Creating a verifier");
// ensure block import and verifier are linked correctly.
let data = maybe_link.as_ref().expect("babe link always provided to verifier instantiation");
let data = maybe_link
.as_ref()
.expect("babe link always provided to verifier instantiation");
let (_, longest_chain) = TestClientBuilder::new().build_with_longest_chain();
@@ -369,10 +368,7 @@ impl TestNetFactory for BabeTestNet {
&self.peers
}
fn mut_peers<F: FnOnce(&mut Vec<BabePeer>)>(
&mut self,
closure: F,
) {
fn mut_peers<F: FnOnce(&mut Vec<BabePeer>)>(&mut self, closure: F) {
closure(&mut self.peers);
}
}
@@ -382,9 +378,7 @@ impl TestNetFactory for BabeTestNet {
fn rejects_empty_block() {
sp_tracing::try_init_simple();
let mut net = BabeTestNet::new(3);
let block_builder = |builder: BlockBuilder<_, _, _>| {
builder.build().unwrap().block
};
let block_builder = |builder: BlockBuilder<_, _, _>| builder.build().unwrap().block;
net.mut_peers(|peer| {
peer[0].generate_blocks(1, BlockOrigin::NetworkInitialSync, block_builder);
})
@@ -397,11 +391,7 @@ fn run_one_test(mutator: impl Fn(&mut TestHeader, Stage) + Send + Sync + 'static
MUTATOR.with(|m| *m.borrow_mut() = mutator.clone());
let net = BabeTestNet::new(3);
let peers = &[
(0, "//Alice"),
(1, "//Bob"),
(2, "//Charlie"),
];
let peers = &[(0, "//Alice"), (1, "//Bob"), (2, "//Charlie")];
let net = Arc::new(Mutex::new(net));
let mut import_notifications = Vec::new();
@@ -415,9 +405,10 @@ fn run_one_test(mutator: impl Fn(&mut TestHeader, Stage) + Send + Sync + 'static
let select_chain = peer.select_chain().expect("Full client has select_chain");
let keystore_path = tempfile::tempdir().expect("Creates keystore path");
let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::open(keystore_path.path(), None)
.expect("Creates keystore"));
SyncCryptoStore::sr25519_generate_new(&*keystore, BABE, Some(seed)).expect("Generates authority key");
let keystore: SyncCryptoStorePtr =
Arc::new(LocalKeystore::open(keystore_path.path(), None).expect("Creates keystore"));
SyncCryptoStore::sr25519_generate_new(&*keystore, BABE, Some(seed))
.expect("Generates authority key");
keystore_paths.push(keystore_path);
let mut got_own = false;
@@ -435,47 +426,54 @@ fn run_one_test(mutator: impl Fn(&mut TestHeader, Stage) + Send + Sync + 'static
import_notifications.push(
// run each future until we get one of our own blocks with number higher than 5
// that was produced locally.
client.import_notification_stream()
.take_while(move |n| future::ready(n.header.number() < &5 || {
if n.origin == BlockOrigin::Own {
got_own = true;
} else {
got_other = true;
}
client
.import_notification_stream()
.take_while(move |n| {
future::ready(
n.header.number() < &5 || {
if n.origin == BlockOrigin::Own {
got_own = true;
} else {
got_other = true;
}
// continue until we have at least one block of our own
// and one of another peer.
!(got_own && got_other)
}))
.for_each(|_| future::ready(()) )
// continue until we have at least one block of our own
// and one of another peer.
!(got_own && got_other)
},
)
})
.for_each(|_| future::ready(())),
);
babe_futures.push(
start_babe(BabeParams {
block_import: data.block_import.lock().take().expect("import set up during init"),
select_chain,
client,
env: environ,
sync_oracle: DummyOracle,
create_inherent_data_providers: Box::new(|_, _| async {
let timestamp = TimestampInherentDataProvider::from_system_time();
let slot = InherentDataProvider::from_timestamp_and_duration(
*timestamp,
Duration::from_secs(6),
);
babe_futures.push(start_babe(BabeParams {
block_import: data.block_import.lock().take().expect("import set up during init"),
select_chain,
client,
env: environ,
sync_oracle: DummyOracle,
create_inherent_data_providers: Box::new(|_, _| async {
let timestamp = TimestampInherentDataProvider::from_system_time();
let slot = InherentDataProvider::from_timestamp_and_duration(
*timestamp,
Duration::from_secs(6),
);
Ok((timestamp, slot))
}),
force_authoring: false,
backoff_authoring_blocks: Some(BackoffAuthoringOnFinalizedHeadLagging::default()),
babe_link: data.link.clone(),
keystore,
can_author_with: sp_consensus::AlwaysCanAuthor,
justification_sync_link: (),
block_proposal_slot_portion: SlotProportion::new(0.5),
max_block_proposal_slot_portion: None,
telemetry: None,
}).expect("Starts babe"));
Ok((timestamp, slot))
}),
force_authoring: false,
backoff_authoring_blocks: Some(BackoffAuthoringOnFinalizedHeadLagging::default()),
babe_link: data.link.clone(),
keystore,
can_author_with: sp_consensus::AlwaysCanAuthor,
justification_sync_link: (),
block_proposal_slot_portion: SlotProportion::new(0.5),
max_block_proposal_slot_portion: None,
telemetry: None,
})
.expect("Starts babe"),
);
}
block_on(future::select(
futures::future::poll_fn(move |cx| {
@@ -489,7 +487,7 @@ fn run_one_test(mutator: impl Fn(&mut TestHeader, Stage) + Send + Sync + 'static
Poll::<()>::Pending
}),
future::select(future::join_all(import_notifications), future::join_all(babe_futures))
future::select(future::join_all(import_notifications), future::join_all(babe_futures)),
));
}
@@ -503,7 +501,8 @@ fn authoring_blocks() {
fn rejects_missing_inherent_digest() {
run_one_test(|header: &mut TestHeader, stage| {
let v = std::mem::take(&mut header.digest_mut().logs);
header.digest_mut().logs = v.into_iter()
header.digest_mut().logs = v
.into_iter()
.filter(|v| stage == Stage::PostSeal || v.as_babe_pre_digest().is_none())
.collect()
})
@@ -514,7 +513,8 @@ fn rejects_missing_inherent_digest() {
fn rejects_missing_seals() {
run_one_test(|header: &mut TestHeader, stage| {
let v = std::mem::take(&mut header.digest_mut().logs);
header.digest_mut().logs = v.into_iter()
header.digest_mut().logs = v
.into_iter()
.filter(|v| stage == Stage::PreSeal || v.as_babe_seal().is_none())
.collect()
})
@@ -525,7 +525,8 @@ fn rejects_missing_seals() {
fn rejects_missing_consensus_digests() {
run_one_test(|header: &mut TestHeader, stage| {
let v = std::mem::take(&mut header.digest_mut().logs);
header.digest_mut().logs = v.into_iter()
header.digest_mut().logs = v
.into_iter()
.filter(|v| stage == Stage::PostSeal || v.as_next_epoch_descriptor().is_none())
.collect()
});
@@ -560,8 +561,8 @@ fn sig_is_not_pre_digest() {
fn can_author_block() {
sp_tracing::try_init_simple();
let keystore_path = tempfile::tempdir().expect("Creates keystore path");
let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::open(keystore_path.path(), None)
.expect("Creates keystore"));
let keystore: SyncCryptoStorePtr =
Arc::new(LocalKeystore::open(keystore_path.path(), None).expect("Creates keystore"));
let public = SyncCryptoStore::sr25519_generate_new(&*keystore, BABE, Some("//Alice"))
.expect("Generates authority pair");
@@ -601,8 +602,8 @@ fn can_author_block() {
None => i += 1,
Some(s) => {
debug!(target: "babe", "Authored block {:?}", s.0);
break;
}
break
},
}
}
}
@@ -622,26 +623,27 @@ fn propose_and_import_block<Transaction: Send + 'static>(
});
let pre_digest = sp_runtime::generic::Digest {
logs: vec![
Item::babe_pre_digest(
PreDigest::SecondaryPlain(SecondaryPlainPreDigest {
authority_index: 0,
slot,
}),
),
],
logs: vec![Item::babe_pre_digest(PreDigest::SecondaryPlain(SecondaryPlainPreDigest {
authority_index: 0,
slot,
}))],
};
let parent_hash = parent.hash();
let mut block = futures::executor::block_on(proposer.propose_with(pre_digest)).unwrap().block;
let epoch_descriptor = proposer_factory.epoch_changes.shared_data().epoch_descriptor_for_child_of(
descendent_query(&*proposer_factory.client),
&parent_hash,
*parent.number(),
slot,
).unwrap().unwrap();
let epoch_descriptor = proposer_factory
.epoch_changes
.shared_data()
.epoch_descriptor_for_child_of(
descendent_query(&*proposer_factory.client),
&parent_hash,
*parent.number(),
slot,
)
.unwrap()
.unwrap();
let seal = {
// sign the pre-sealed hash of the block and then
@@ -706,13 +708,12 @@ fn importing_block_one_sets_genesis_epoch() {
let genesis_epoch = Epoch::genesis(&data.link.config, 999.into());
let epoch_changes = data.link.epoch_changes.shared_data();
let epoch_for_second_block = epoch_changes.epoch_data_for_child_of(
descendent_query(&*client),
&block_hash,
1,
1000.into(),
|slot| Epoch::genesis(&data.link.config, slot),
).unwrap().unwrap();
let epoch_for_second_block = epoch_changes
.epoch_data_for_child_of(descendent_query(&*client), &block_hash, 1, 1000.into(), |slot| {
Epoch::genesis(&data.link.config, slot)
})
.unwrap()
.unwrap();
assert_eq!(epoch_for_second_block, genesis_epoch);
}
@@ -779,16 +780,10 @@ fn importing_epoch_change_block_prunes_tree() {
let fork_3 = propose_and_import_blocks(BlockId::Hash(canon_hashes[18]), 10);
// We should be tracking a total of 9 epochs in the fork tree
assert_eq!(
epoch_changes.shared_data().tree().iter().count(),
9,
);
assert_eq!(epoch_changes.shared_data().tree().iter().count(), 9,);
// And only one root
assert_eq!(
epoch_changes.shared_data().tree().roots().count(),
1,
);
assert_eq!(epoch_changes.shared_data().tree().roots().count(), 1,);
// We finalize block #13 from the canon chain, so on the next epoch
// change the tree should be pruned, to not contain F (#7).
@@ -796,32 +791,47 @@ fn importing_epoch_change_block_prunes_tree() {
propose_and_import_blocks(BlockId::Hash(client.chain_info().best_hash), 7);
// at this point no hashes from the first fork must exist on the tree
assert!(
!epoch_changes.shared_data().tree().iter().map(|(h, _, _)| h).any(|h| fork_1.contains(h)),
);
assert!(!epoch_changes
.shared_data()
.tree()
.iter()
.map(|(h, _, _)| h)
.any(|h| fork_1.contains(h)),);
// but the epoch changes from the other forks must still exist
assert!(
epoch_changes.shared_data().tree().iter().map(|(h, _, _)| h).any(|h| fork_2.contains(h))
);
assert!(epoch_changes
.shared_data()
.tree()
.iter()
.map(|(h, _, _)| h)
.any(|h| fork_2.contains(h)));
assert!(
epoch_changes.shared_data().tree().iter().map(|(h, _, _)| h).any(|h| fork_3.contains(h)),
);
assert!(epoch_changes
.shared_data()
.tree()
.iter()
.map(|(h, _, _)| h)
.any(|h| fork_3.contains(h)),);
// finalizing block #25 from the canon chain should prune out the second fork
client.finalize_block(BlockId::Hash(canon_hashes[24]), None, false).unwrap();
propose_and_import_blocks(BlockId::Hash(client.chain_info().best_hash), 8);
// at this point no hashes from the second fork must exist on the tree
assert!(
!epoch_changes.shared_data().tree().iter().map(|(h, _, _)| h).any(|h| fork_2.contains(h)),
);
assert!(!epoch_changes
.shared_data()
.tree()
.iter()
.map(|(h, _, _)| h)
.any(|h| fork_2.contains(h)),);
// while epoch changes from the last fork should still exist
assert!(
epoch_changes.shared_data().tree().iter().map(|(h, _, _)| h).any(|h| fork_3.contains(h)),
);
assert!(epoch_changes
.shared_data()
.tree()
.iter()
.map(|(h, _, _)| h)
.any(|h| fork_3.contains(h)),);
}
#[test]
@@ -856,20 +866,15 @@ fn verify_slots_are_strictly_increasing() {
// we should fail to import this block since the slot number didn't increase.
// we will panic due to the `PanickingBlockImport` defined above.
propose_and_import_block(
&b1,
Some(999.into()),
&mut proposer_factory,
&mut block_import,
);
propose_and_import_block(&b1, Some(999.into()), &mut proposer_factory, &mut block_import);
}
#[test]
fn babe_transcript_generation_match() {
sp_tracing::try_init_simple();
let keystore_path = tempfile::tempdir().expect("Creates keystore path");
let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::open(keystore_path.path(), None)
.expect("Creates keystore"));
let keystore: SyncCryptoStorePtr =
Arc::new(LocalKeystore::open(keystore_path.path(), None).expect("Creates keystore"));
let public = SyncCryptoStore::sr25519_generate_new(&*keystore, BABE, Some("//Alice"))
.expect("Generates authority pair");
@@ -890,9 +895,7 @@ fn babe_transcript_generation_match() {
let test = |t: merlin::Transcript| -> [u8; 16] {
let mut b = [0u8; 16];
t.build_rng()
.finalize(&mut ChaChaRng::from_seed([0u8;32]))
.fill_bytes(&mut b);
t.build_rng().finalize(&mut ChaChaRng::from_seed([0u8; 32])).fill_bytes(&mut b);
b
};
debug_assert!(test(orig_transcript) == test(transcript_from_data(new_transcript)));
@@ -17,18 +17,22 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//! Verification for BABE headers.
use sp_runtime::{traits::Header, traits::DigestItemFor};
use sp_core::{Pair, Public};
use sp_consensus_babe::{make_transcript, AuthoritySignature, AuthorityPair, AuthorityId};
use sp_consensus_babe::digests::{
PreDigest, PrimaryPreDigest, SecondaryPlainPreDigest, SecondaryVRFPreDigest,
CompatibleDigestItem
use super::{
authorship::{calculate_primary_threshold, check_primary_threshold, secondary_slot_author},
babe_err, find_pre_digest, BlockT, Epoch, Error,
};
use sc_consensus_slots::CheckedHeader;
use sp_consensus_slots::Slot;
use log::{debug, trace};
use super::{find_pre_digest, babe_err, Epoch, BlockT, Error};
use super::authorship::{calculate_primary_threshold, check_primary_threshold, secondary_slot_author};
use sc_consensus_slots::CheckedHeader;
use sp_consensus_babe::{
digests::{
CompatibleDigestItem, PreDigest, PrimaryPreDigest, SecondaryPlainPreDigest,
SecondaryVRFPreDigest,
},
make_transcript, AuthorityId, AuthorityPair, AuthoritySignature,
};
use sp_consensus_slots::Slot;
use sp_core::{Pair, Public};
use sp_runtime::traits::{DigestItemFor, Header};
/// BABE verification parameters
pub(super) struct VerificationParams<'a, B: 'a + BlockT> {
@@ -57,26 +61,24 @@ pub(super) struct VerificationParams<'a, B: 'a + BlockT> {
/// with each having different validation logic.
pub(super) fn check_header<B: BlockT + Sized>(
params: VerificationParams<B>,
) -> Result<CheckedHeader<B::Header, VerifiedHeaderInfo<B>>, Error<B>> where
) -> Result<CheckedHeader<B::Header, VerifiedHeaderInfo<B>>, Error<B>>
where
DigestItemFor<B>: CompatibleDigestItem,
{
let VerificationParams {
mut header,
pre_digest,
slot_now,
epoch,
} = params;
let VerificationParams { mut header, pre_digest, slot_now, epoch } = params;
let authorities = &epoch.authorities;
let pre_digest = pre_digest.map(Ok).unwrap_or_else(|| find_pre_digest::<B>(&header))?;
trace!(target: "babe", "Checking header");
let seal = header.digest_mut().pop()
let seal = header
.digest_mut()
.pop()
.ok_or_else(|| babe_err(Error::HeaderUnsealed(header.hash())))?;
let sig = seal.as_babe_seal().ok_or_else(|| {
babe_err(Error::HeaderBadSeal(header.hash()))
})?;
let sig = seal
.as_babe_seal()
.ok_or_else(|| babe_err(Error::HeaderBadSeal(header.hash())))?;
// the pre-hash of the header doesn't include the seal
// and that's what we sign
@@ -84,7 +86,7 @@ pub(super) fn check_header<B: BlockT + Sized>(
if pre_digest.slot() > slot_now {
header.digest_mut().push(seal);
return Ok(CheckedHeader::Deferred(header, pre_digest.slot()));
return Ok(CheckedHeader::Deferred(header, pre_digest.slot()))
}
let author = match authorities.get(pre_digest.authority_index() as usize) {
@@ -100,45 +102,31 @@ pub(super) fn check_header<B: BlockT + Sized>(
primary.slot,
);
check_primary_header::<B>(
pre_hash,
primary,
sig,
&epoch,
epoch.config.c,
)?;
check_primary_header::<B>(pre_hash, primary, sig, &epoch, epoch.config.c)?;
},
PreDigest::SecondaryPlain(secondary) if epoch.config.allowed_slots.is_secondary_plain_slots_allowed() => {
PreDigest::SecondaryPlain(secondary)
if epoch.config.allowed_slots.is_secondary_plain_slots_allowed() =>
{
debug!(target: "babe",
"Verifying secondary plain block #{} at slot: {}",
header.number(),
secondary.slot,
);
check_secondary_plain_header::<B>(
pre_hash,
secondary,
sig,
&epoch,
)?;
},
PreDigest::SecondaryVRF(secondary) if epoch.config.allowed_slots.is_secondary_vrf_slots_allowed() => {
check_secondary_plain_header::<B>(pre_hash, secondary, sig, &epoch)?;
}
PreDigest::SecondaryVRF(secondary)
if epoch.config.allowed_slots.is_secondary_vrf_slots_allowed() =>
{
debug!(target: "babe",
"Verifying secondary VRF block #{} at slot: {}",
header.number(),
secondary.slot,
);
check_secondary_vrf_header::<B>(
pre_hash,
secondary,
sig,
&epoch,
)?;
},
_ => {
return Err(babe_err(Error::SecondarySlotAssignmentsDisabled));
check_secondary_vrf_header::<B>(pre_hash, secondary, sig, &epoch)?;
}
_ => return Err(babe_err(Error::SecondarySlotAssignmentsDisabled)),
}
let info = VerifiedHeaderInfo {
@@ -170,27 +158,20 @@ fn check_primary_header<B: BlockT + Sized>(
if AuthorityPair::verify(&signature, pre_hash, &author) {
let (inout, _) = {
let transcript = make_transcript(
&epoch.randomness,
pre_digest.slot,
epoch.epoch_index,
);
let transcript = make_transcript(&epoch.randomness, pre_digest.slot, epoch.epoch_index);
schnorrkel::PublicKey::from_bytes(author.as_slice()).and_then(|p| {
p.vrf_verify(transcript, &pre_digest.vrf_output, &pre_digest.vrf_proof)
}).map_err(|s| {
babe_err(Error::VRFVerificationFailed(s))
})?
schnorrkel::PublicKey::from_bytes(author.as_slice())
.and_then(|p| {
p.vrf_verify(transcript, &pre_digest.vrf_output, &pre_digest.vrf_proof)
})
.map_err(|s| babe_err(Error::VRFVerificationFailed(s)))?
};
let threshold = calculate_primary_threshold(
c,
&epoch.authorities,
pre_digest.authority_index as usize,
);
let threshold =
calculate_primary_threshold(c, &epoch.authorities, pre_digest.authority_index as usize);
if !check_primary_threshold(&inout, threshold) {
return Err(babe_err(Error::VRFVerificationOfBlockFailed(author.clone(), threshold)));
return Err(babe_err(Error::VRFVerificationOfBlockFailed(author.clone(), threshold)))
}
Ok(())
@@ -211,16 +192,14 @@ fn check_secondary_plain_header<B: BlockT>(
) -> Result<(), Error<B>> {
// check the signature is valid under the expected authority and
// chain state.
let expected_author = secondary_slot_author(
pre_digest.slot,
&epoch.authorities,
epoch.randomness,
).ok_or_else(|| Error::NoSecondaryAuthorExpected)?;
let expected_author =
secondary_slot_author(pre_digest.slot, &epoch.authorities, epoch.randomness)
.ok_or_else(|| Error::NoSecondaryAuthorExpected)?;
let author = &epoch.authorities[pre_digest.authority_index as usize].0;
if expected_author != author {
return Err(Error::InvalidAuthor(expected_author.clone(), author.clone()));
return Err(Error::InvalidAuthor(expected_author.clone(), author.clone()))
}
if AuthorityPair::verify(&signature, pre_hash.as_ref(), author) {
@@ -239,30 +218,22 @@ fn check_secondary_vrf_header<B: BlockT>(
) -> Result<(), Error<B>> {
// check the signature is valid under the expected authority and
// chain state.
let expected_author = secondary_slot_author(
pre_digest.slot,
&epoch.authorities,
epoch.randomness,
).ok_or_else(|| Error::NoSecondaryAuthorExpected)?;
let expected_author =
secondary_slot_author(pre_digest.slot, &epoch.authorities, epoch.randomness)
.ok_or_else(|| Error::NoSecondaryAuthorExpected)?;
let author = &epoch.authorities[pre_digest.authority_index as usize].0;
if expected_author != author {
return Err(Error::InvalidAuthor(expected_author.clone(), author.clone()));
return Err(Error::InvalidAuthor(expected_author.clone(), author.clone()))
}
if AuthorityPair::verify(&signature, pre_hash.as_ref(), author) {
let transcript = make_transcript(
&epoch.randomness,
pre_digest.slot,
epoch.epoch_index,
);
let transcript = make_transcript(&epoch.randomness, pre_digest.slot, epoch.epoch_index);
schnorrkel::PublicKey::from_bytes(author.as_slice()).and_then(|p| {
p.vrf_verify(transcript, &pre_digest.vrf_output, &pre_digest.vrf_proof)
}).map_err(|s| {
babe_err(Error::VRFVerificationFailed(s))
})?;
schnorrkel::PublicKey::from_bytes(author.as_slice())
.and_then(|p| p.vrf_verify(transcript, &pre_digest.vrf_output, &pre_digest.vrf_proof))
.map_err(|s| babe_err(Error::VRFVerificationFailed(s)))?;
Ok(())
} else {
@@ -18,30 +18,26 @@
//! Longest chain implementation
use std::sync::Arc;
use std::marker::PhantomData;
use sc_client_api::backend;
use sp_consensus::{SelectChain, Error as ConsensusError};
use sp_blockchain::{Backend, HeaderBackend};
use sp_consensus::{Error as ConsensusError, SelectChain};
use sp_runtime::{
traits::{NumberFor, Block as BlockT},
generic::BlockId,
traits::{Block as BlockT, NumberFor},
};
use std::{marker::PhantomData, sync::Arc};
/// Implement Longest Chain Select implementation
/// where 'longest' is defined as the highest number of blocks
pub struct LongestChain<B, Block> {
backend: Arc<B>,
_phantom: PhantomData<Block>
_phantom: PhantomData<Block>,
}
impl<B, Block> Clone for LongestChain<B, Block> {
fn clone(&self) -> Self {
let backend = self.backend.clone();
LongestChain {
backend,
_phantom: Default::default()
}
LongestChain { backend, _phantom: Default::default() }
}
}
@@ -52,21 +48,22 @@ where
{
/// Instantiate a new LongestChain for Backend B
pub fn new(backend: Arc<B>) -> Self {
LongestChain {
backend,
_phantom: Default::default(),
}
LongestChain { backend, _phantom: Default::default() }
}
fn best_block_header(&self) -> sp_blockchain::Result<<Block as BlockT>::Header> {
let info = self.backend.blockchain().info();
let import_lock = self.backend.get_import_lock();
let best_hash = self.backend
let best_hash = self
.backend
.blockchain()
.best_containing(info.best_hash, None, import_lock)?
.unwrap_or(info.best_hash);
Ok(self.backend.blockchain().header(BlockId::Hash(best_hash))?
Ok(self
.backend
.blockchain()
.header(BlockId::Hash(best_hash))?
.expect("given block hash was fetched from block in db; qed"))
}
@@ -18,8 +18,8 @@
//! Provides a generic wrapper around shared data. See [`SharedData`] for more information.
use parking_lot::{Condvar, MappedMutexGuard, Mutex, MutexGuard};
use std::sync::Arc;
use parking_lot::{Mutex, MappedMutexGuard, Condvar, MutexGuard};
/// Created by [`SharedDataLocked::release_mutex`].
///
@@ -75,8 +75,7 @@ impl<'a, T> SharedDataLocked<'a, T> {
/// Release the mutex, but keep the shared data locked.
pub fn release_mutex(mut self) -> SharedDataLockedUpgradable<T> {
SharedDataLockedUpgradable {
shared_data: self.shared_data.take()
.expect("`shared_data` is only taken on drop; qed"),
shared_data: self.shared_data.take().expect("`shared_data` is only taken on drop; qed"),
}
}
}
@@ -132,7 +131,7 @@ struct SharedDataInner<T> {
/// # Example
///
/// ```
///# use sc_consensus::shared_data::SharedData;
/// # use sc_consensus::shared_data::SharedData;
///
/// let shared_data = SharedData::new(String::from("hello world"));
///
@@ -174,10 +173,7 @@ pub struct SharedData<T> {
impl<T> Clone for SharedData<T> {
fn clone(&self) -> Self {
Self {
inner: self.inner.clone(),
cond_var: self.cond_var.clone(),
}
Self { inner: self.inner.clone(), cond_var: self.cond_var.clone() }
}
}
@@ -228,10 +224,7 @@ impl<T> SharedData<T> {
debug_assert!(!guard.locked);
guard.locked = true;
SharedDataLocked {
inner: guard,
shared_data: Some(self.clone()),
}
SharedDataLocked { inner: guard, shared_data: Some(self.clone()) }
}
}
+213 -290
View File
@@ -20,12 +20,16 @@
pub mod migration;
use std::{ops::Add, collections::BTreeMap, borrow::{Borrow, BorrowMut}};
use codec::{Encode, Decode};
use codec::{Decode, Encode};
use fork_tree::ForkTree;
use sc_client_api::utils::is_descendent_of;
use sp_blockchain::{HeaderMetadata, HeaderBackend, Error as ClientError};
use sp_blockchain::{Error as ClientError, HeaderBackend, HeaderMetadata};
use sp_runtime::traits::{Block as BlockT, NumberFor, One, Zero};
use std::{
borrow::{Borrow, BorrowMut},
collections::BTreeMap,
ops::Add,
};
/// A builder for `is_descendent_of` functions.
pub trait IsDescendentOfBuilder<Hash> {
@@ -41,8 +45,7 @@ pub trait IsDescendentOfBuilder<Hash> {
/// details aren't yet stored, but its parent is.
///
/// The format of `current` when `Some` is `(current, current_parent)`.
fn build_is_descendent_of(&self, current: Option<(Hash, Hash)>)
-> Self::IsDescendentOf;
fn build_is_descendent_of(&self, current: Option<(Hash, Hash)>) -> Self::IsDescendentOf;
}
/// Produce a descendent query object given the client.
@@ -55,16 +58,18 @@ pub fn descendent_query<H, Block>(client: &H) -> HeaderBackendDescendentBuilder<
pub struct HeaderBackendDescendentBuilder<H, Block>(H, std::marker::PhantomData<Block>);
impl<'a, H, Block> IsDescendentOfBuilder<Block::Hash>
for HeaderBackendDescendentBuilder<&'a H, Block> where
H: HeaderBackend<Block> + HeaderMetadata<Block, Error=ClientError>,
for HeaderBackendDescendentBuilder<&'a H, Block>
where
H: HeaderBackend<Block> + HeaderMetadata<Block, Error = ClientError>,
Block: BlockT,
{
type Error = ClientError;
type IsDescendentOf = Box<dyn Fn(&Block::Hash, &Block::Hash) -> Result<bool, ClientError> + 'a>;
fn build_is_descendent_of(&self, current: Option<(Block::Hash, Block::Hash)>)
-> Self::IsDescendentOf
{
fn build_is_descendent_of(
&self,
current: Option<(Block::Hash, Block::Hash)>,
) -> Self::IsDescendentOf {
Box::new(is_descendent_of(self.0, current))
}
}
@@ -90,10 +95,7 @@ pub trait Epoch {
impl<'a, E: Epoch> From<&'a E> for EpochHeader<E> {
fn from(epoch: &'a E) -> EpochHeader<E> {
Self {
start_slot: epoch.start_slot(),
end_slot: epoch.end_slot(),
}
Self { start_slot: epoch.start_slot(), end_slot: epoch.end_slot() }
}
}
@@ -109,10 +111,7 @@ pub struct EpochHeader<E: Epoch> {
impl<E: Epoch> Clone for EpochHeader<E> {
fn clone(&self) -> Self {
Self {
start_slot: self.start_slot,
end_slot: self.end_slot,
}
Self { start_slot: self.start_slot, end_slot: self.end_slot }
}
}
@@ -149,7 +148,8 @@ pub enum ViableEpoch<E, ERef = E> {
Signaled(ERef),
}
impl<E, ERef> AsRef<E> for ViableEpoch<E, ERef> where
impl<E, ERef> AsRef<E> for ViableEpoch<E, ERef>
where
ERef: Borrow<E>,
{
fn as_ref(&self) -> &E {
@@ -160,7 +160,8 @@ impl<E, ERef> AsRef<E> for ViableEpoch<E, ERef> where
}
}
impl<E, ERef> AsMut<E> for ViableEpoch<E, ERef> where
impl<E, ERef> AsMut<E> for ViableEpoch<E, ERef>
where
ERef: BorrowMut<E>,
{
fn as_mut(&mut self) -> &mut E {
@@ -171,7 +172,8 @@ impl<E, ERef> AsMut<E> for ViableEpoch<E, ERef> where
}
}
impl<E, ERef> ViableEpoch<E, ERef> where
impl<E, ERef> ViableEpoch<E, ERef>
where
E: Epoch + Clone,
ERef: Borrow<E>,
{
@@ -187,18 +189,14 @@ impl<E, ERef> ViableEpoch<E, ERef> where
/// Get cloned value for the viable epoch.
pub fn into_cloned(self) -> ViableEpoch<E, E> {
match self {
ViableEpoch::UnimportedGenesis(e) =>
ViableEpoch::UnimportedGenesis(e),
ViableEpoch::UnimportedGenesis(e) => ViableEpoch::UnimportedGenesis(e),
ViableEpoch::Signaled(e) => ViableEpoch::Signaled(e.borrow().clone()),
}
}
/// Increment the epoch, yielding an `IncrementedEpoch` to be imported
/// into the fork-tree.
pub fn increment(
&self,
next_descriptor: E::NextEpochDescriptor
) -> IncrementedEpoch<E> {
pub fn increment(&self, next_descriptor: E::NextEpochDescriptor) -> IncrementedEpoch<E> {
let next = self.as_ref().increment(next_descriptor);
let to_persist = match *self {
ViableEpoch::UnimportedGenesis(ref epoch_0) =>
@@ -216,7 +214,7 @@ pub enum ViableEpochDescriptor<Hash, Number, E: Epoch> {
/// The epoch is an unimported genesis, with given start slot number.
UnimportedGenesis(E::Slot),
/// The epoch is signaled and has been imported, with given identifier and header.
Signaled(EpochIdentifier<Hash, Number>, EpochHeader<E>)
Signaled(EpochIdentifier<Hash, Number>, EpochHeader<E>),
}
impl<Hash, Number, E: Epoch> ViableEpochDescriptor<Hash, Number, E> {
@@ -243,8 +241,7 @@ impl<'a, E: Epoch> From<&'a PersistedEpoch<E>> for PersistedEpochHeader<E> {
match epoch {
PersistedEpoch::Genesis(ref epoch_0, ref epoch_1) =>
PersistedEpochHeader::Genesis(epoch_0.into(), epoch_1.into()),
PersistedEpoch::Regular(ref epoch_n) =>
PersistedEpochHeader::Regular(epoch_n.into()),
PersistedEpoch::Regular(ref epoch_n) => PersistedEpochHeader::Regular(epoch_n.into()),
}
}
}
@@ -312,7 +309,8 @@ fn fake_head_hash<H: AsRef<[u8]> + AsMut<[u8]> + Clone>(parent_hash: &H) -> H {
h
}
impl<Hash, Number, E: Epoch> Default for EpochChanges<Hash, Number, E> where
impl<Hash, Number, E: Epoch> Default for EpochChanges<Hash, Number, E>
where
Hash: PartialEq + Ord,
Number: Ord,
{
@@ -321,9 +319,10 @@ impl<Hash, Number, E: Epoch> Default for EpochChanges<Hash, Number, E> where
}
}
impl<Hash, Number, E: Epoch> EpochChanges<Hash, Number, E> where
impl<Hash, Number, E: Epoch> EpochChanges<Hash, Number, E>
where
Hash: PartialEq + Ord + AsRef<[u8]> + AsMut<[u8]> + Copy,
Number: Ord + One + Zero + Add<Output=Number> + Copy,
Number: Ord + One + Zero + Add<Output = Number> + Copy,
{
/// Create a new epoch change.
pub fn new() -> Self {
@@ -337,51 +336,38 @@ impl<Hash, Number, E: Epoch> EpochChanges<Hash, Number, E> where
}
/// Map the epoch changes from one storing data to a different one.
pub fn map<B, F>(self, mut f: F) -> EpochChanges<Hash, Number, B> where
B: Epoch<Slot=E::Slot>,
pub fn map<B, F>(self, mut f: F) -> EpochChanges<Hash, Number, B>
where
B: Epoch<Slot = E::Slot>,
F: FnMut(&Hash, &Number, E) -> B,
{
EpochChanges {
inner: self.inner.map(&mut |_, _, header| {
match header {
PersistedEpochHeader::Genesis(epoch_0, epoch_1) => {
PersistedEpochHeader::Genesis(
EpochHeader {
start_slot: epoch_0.start_slot,
end_slot: epoch_0.end_slot,
},
EpochHeader {
start_slot: epoch_1.start_slot,
end_slot: epoch_1.end_slot,
},
)
},
PersistedEpochHeader::Regular(epoch_n) => {
PersistedEpochHeader::Regular(
EpochHeader {
start_slot: epoch_n.start_slot,
end_slot: epoch_n.end_slot,
},
)
},
}
inner: self.inner.map(&mut |_, _, header| match header {
PersistedEpochHeader::Genesis(epoch_0, epoch_1) => PersistedEpochHeader::Genesis(
EpochHeader { start_slot: epoch_0.start_slot, end_slot: epoch_0.end_slot },
EpochHeader { start_slot: epoch_1.start_slot, end_slot: epoch_1.end_slot },
),
PersistedEpochHeader::Regular(epoch_n) =>
PersistedEpochHeader::Regular(EpochHeader {
start_slot: epoch_n.start_slot,
end_slot: epoch_n.end_slot,
}),
}),
epochs: self.epochs.into_iter().map(|((hash, number), epoch)| {
let bepoch = match epoch {
PersistedEpoch::Genesis(epoch_0, epoch_1) => {
PersistedEpoch::Genesis(
epochs: self
.epochs
.into_iter()
.map(|((hash, number), epoch)| {
let bepoch = match epoch {
PersistedEpoch::Genesis(epoch_0, epoch_1) => PersistedEpoch::Genesis(
f(&hash, &number, epoch_0),
f(&hash, &number, epoch_1),
)
},
PersistedEpoch::Regular(epoch_n) => {
PersistedEpoch::Regular(
f(&hash, &number, epoch_n)
)
},
};
((hash, number), bepoch)
}).collect(),
),
PersistedEpoch::Regular(epoch_n) =>
PersistedEpoch::Regular(f(&hash, &number, epoch_n)),
};
((hash, number), bepoch)
})
.collect(),
}
}
@@ -395,25 +381,17 @@ impl<Hash, Number, E: Epoch> EpochChanges<Hash, Number, E> where
number: Number,
slot: E::Slot,
) -> Result<(), fork_tree::Error<D::Error>> {
let is_descendent_of = descendent_of_builder
.build_is_descendent_of(None);
let is_descendent_of = descendent_of_builder.build_is_descendent_of(None);
let predicate = |epoch: &PersistedEpochHeader<E>| match *epoch {
PersistedEpochHeader::Genesis(_, ref epoch_1) =>
slot >= epoch_1.end_slot,
PersistedEpochHeader::Regular(ref epoch_n) =>
slot >= epoch_n.end_slot,
PersistedEpochHeader::Genesis(_, ref epoch_1) => slot >= epoch_1.end_slot,
PersistedEpochHeader::Regular(ref epoch_n) => slot >= epoch_n.end_slot,
};
// prune any epochs which could not be _live_ as of the children of the
// finalized block, i.e. re-root the fork tree to the oldest ancestor of
// (hash, number) where epoch.end_slot() >= finalized_slot
let removed = self.inner.prune(
hash,
&number,
&is_descendent_of,
&predicate,
)?;
let removed = self.inner.prune(hash, &number, &is_descendent_of, &predicate)?;
for (hash, number, _) in removed {
self.epochs.remove(&(hash, number));
@@ -424,18 +402,18 @@ impl<Hash, Number, E: Epoch> EpochChanges<Hash, Number, E> where
/// Get a reference to an epoch with given identifier.
pub fn epoch(&self, id: &EpochIdentifier<Hash, Number>) -> Option<&E> {
self.epochs.get(&(id.hash, id.number))
.and_then(|v| {
match v {
PersistedEpoch::Genesis(ref epoch_0, _)
if id.position == EpochIdentifierPosition::Genesis0 => Some(epoch_0),
PersistedEpoch::Genesis(_, ref epoch_1)
if id.position == EpochIdentifierPosition::Genesis1 => Some(epoch_1),
PersistedEpoch::Regular(ref epoch_n)
if id.position == EpochIdentifierPosition::Regular => Some(epoch_n),
_ => None,
}
})
self.epochs.get(&(id.hash, id.number)).and_then(|v| match v {
PersistedEpoch::Genesis(ref epoch_0, _)
if id.position == EpochIdentifierPosition::Genesis0 =>
Some(epoch_0),
PersistedEpoch::Genesis(_, ref epoch_1)
if id.position == EpochIdentifierPosition::Genesis1 =>
Some(epoch_1),
PersistedEpoch::Regular(ref epoch_n)
if id.position == EpochIdentifierPosition::Regular =>
Some(epoch_n),
_ => None,
})
}
/// Get a reference to a viable epoch with given descriptor.
@@ -443,33 +421,32 @@ impl<Hash, Number, E: Epoch> EpochChanges<Hash, Number, E> where
&self,
descriptor: &ViableEpochDescriptor<Hash, Number, E>,
make_genesis: G,
) -> Option<ViableEpoch<E, &E>> where
G: FnOnce(E::Slot) -> E
) -> Option<ViableEpoch<E, &E>>
where
G: FnOnce(E::Slot) -> E,
{
match descriptor {
ViableEpochDescriptor::UnimportedGenesis(slot) => {
Some(ViableEpoch::UnimportedGenesis(make_genesis(*slot)))
},
ViableEpochDescriptor::Signaled(identifier, _) => {
self.epoch(&identifier).map(ViableEpoch::Signaled)
},
ViableEpochDescriptor::UnimportedGenesis(slot) =>
Some(ViableEpoch::UnimportedGenesis(make_genesis(*slot))),
ViableEpochDescriptor::Signaled(identifier, _) =>
self.epoch(&identifier).map(ViableEpoch::Signaled),
}
}
/// Get a mutable reference to an epoch with given identifier.
pub fn epoch_mut(&mut self, id: &EpochIdentifier<Hash, Number>) -> Option<&mut E> {
self.epochs.get_mut(&(id.hash, id.number))
.and_then(|v| {
match v {
PersistedEpoch::Genesis(ref mut epoch_0, _)
if id.position == EpochIdentifierPosition::Genesis0 => Some(epoch_0),
PersistedEpoch::Genesis(_, ref mut epoch_1)
if id.position == EpochIdentifierPosition::Genesis1 => Some(epoch_1),
PersistedEpoch::Regular(ref mut epoch_n)
if id.position == EpochIdentifierPosition::Regular => Some(epoch_n),
_ => None,
}
})
self.epochs.get_mut(&(id.hash, id.number)).and_then(|v| match v {
PersistedEpoch::Genesis(ref mut epoch_0, _)
if id.position == EpochIdentifierPosition::Genesis0 =>
Some(epoch_0),
PersistedEpoch::Genesis(_, ref mut epoch_1)
if id.position == EpochIdentifierPosition::Genesis1 =>
Some(epoch_1),
PersistedEpoch::Regular(ref mut epoch_n)
if id.position == EpochIdentifierPosition::Regular =>
Some(epoch_n),
_ => None,
})
}
/// Get a mutable reference to a viable epoch with given descriptor.
@@ -477,16 +454,15 @@ impl<Hash, Number, E: Epoch> EpochChanges<Hash, Number, E> where
&mut self,
descriptor: &ViableEpochDescriptor<Hash, Number, E>,
make_genesis: G,
) -> Option<ViableEpoch<E, &mut E>> where
G: FnOnce(E::Slot) -> E
) -> Option<ViableEpoch<E, &mut E>>
where
G: FnOnce(E::Slot) -> E,
{
match descriptor {
ViableEpochDescriptor::UnimportedGenesis(slot) => {
Some(ViableEpoch::UnimportedGenesis(make_genesis(*slot)))
},
ViableEpochDescriptor::Signaled(identifier, _) => {
self.epoch_mut(&identifier).map(ViableEpoch::Signaled)
},
ViableEpochDescriptor::UnimportedGenesis(slot) =>
Some(ViableEpoch::UnimportedGenesis(make_genesis(*slot))),
ViableEpochDescriptor::Signaled(identifier, _) =>
self.epoch_mut(&identifier).map(ViableEpoch::Signaled),
}
}
@@ -497,18 +473,15 @@ impl<Hash, Number, E: Epoch> EpochChanges<Hash, Number, E> where
pub fn epoch_data<G>(
&self,
descriptor: &ViableEpochDescriptor<Hash, Number, E>,
make_genesis: G
) -> Option<E> where
make_genesis: G,
) -> Option<E>
where
G: FnOnce(E::Slot) -> E,
E: Clone,
{
match descriptor {
ViableEpochDescriptor::UnimportedGenesis(slot) => {
Some(make_genesis(*slot))
},
ViableEpochDescriptor::Signaled(identifier, _) => {
self.epoch(&identifier).cloned()
},
ViableEpochDescriptor::UnimportedGenesis(slot) => Some(make_genesis(*slot)),
ViableEpochDescriptor::Signaled(identifier, _) => self.epoch(&identifier).cloned(),
}
}
@@ -524,7 +497,8 @@ impl<Hash, Number, E: Epoch> EpochChanges<Hash, Number, E> where
parent_number: Number,
slot: E::Slot,
make_genesis: G,
) -> Result<Option<E>, fork_tree::Error<D::Error>> where
) -> Result<Option<E>, fork_tree::Error<D::Error>>
where
G: FnOnce(E::Slot) -> E,
E: Clone,
{
@@ -532,7 +506,7 @@ impl<Hash, Number, E: Epoch> EpochChanges<Hash, Number, E> where
descendent_of_builder,
parent_hash,
parent_number,
slot
slot,
)?;
Ok(descriptor.and_then(|des| self.epoch_data(&des, make_genesis)))
@@ -555,8 +529,8 @@ impl<Hash, Number, E: Epoch> EpochChanges<Hash, Number, E> where
// "descends" from our parent-hash.
let fake_head_hash = fake_head_hash(parent_hash);
let is_descendent_of = descendent_of_builder
.build_is_descendent_of(Some((fake_head_hash, *parent_hash)));
let is_descendent_of =
descendent_of_builder.build_is_descendent_of(Some((fake_head_hash, *parent_hash)));
if parent_number == Zero::zero() {
// need to insert the genesis epoch.
@@ -569,37 +543,41 @@ impl<Hash, Number, E: Epoch> EpochChanges<Hash, Number, E> where
// at epoch_1 -- all we're doing here is figuring out which node
// we need.
let predicate = |epoch: &PersistedEpochHeader<E>| match *epoch {
PersistedEpochHeader::Genesis(ref epoch_0, _) =>
epoch_0.start_slot <= slot,
PersistedEpochHeader::Regular(ref epoch_n) =>
epoch_n.start_slot <= slot,
PersistedEpochHeader::Genesis(ref epoch_0, _) => epoch_0.start_slot <= slot,
PersistedEpochHeader::Regular(ref epoch_n) => epoch_n.start_slot <= slot,
};
self.inner.find_node_where(
&fake_head_hash,
&(parent_number + One::one()),
&is_descendent_of,
&predicate,
)
self.inner
.find_node_where(
&fake_head_hash,
&(parent_number + One::one()),
&is_descendent_of,
&predicate,
)
.map(|n| {
n.map(|node| (match node.data {
// Ok, we found our node.
// and here we figure out which of the internal epochs
// of a genesis node to use based on their start slot.
PersistedEpochHeader::Genesis(ref epoch_0, ref epoch_1) =>
if epoch_1.start_slot <= slot {
(EpochIdentifierPosition::Genesis1, epoch_1.clone())
} else {
(EpochIdentifierPosition::Genesis0, epoch_0.clone())
n.map(|node| {
(
match node.data {
// Ok, we found our node.
// and here we figure out which of the internal epochs
// of a genesis node to use based on their start slot.
PersistedEpochHeader::Genesis(ref epoch_0, ref epoch_1) =>
if epoch_1.start_slot <= slot {
(EpochIdentifierPosition::Genesis1, epoch_1.clone())
} else {
(EpochIdentifierPosition::Genesis0, epoch_0.clone())
},
PersistedEpochHeader::Regular(ref epoch_n) =>
(EpochIdentifierPosition::Regular, epoch_n.clone()),
},
PersistedEpochHeader::Regular(ref epoch_n) =>
(EpochIdentifierPosition::Regular, epoch_n.clone()),
}, node)).map(|((position, header), node)| {
ViableEpochDescriptor::Signaled(EpochIdentifier {
position,
hash: node.hash,
number: node.number
}, header)
node,
)
})
.map(|((position, header), node)| {
ViableEpochDescriptor::Signaled(
EpochIdentifier { position, hash: node.hash, number: node.number },
header,
)
})
})
}
@@ -617,16 +595,11 @@ impl<Hash, Number, E: Epoch> EpochChanges<Hash, Number, E> where
parent_hash: Hash,
epoch: IncrementedEpoch<E>,
) -> Result<(), fork_tree::Error<D::Error>> {
let is_descendent_of = descendent_of_builder
.build_is_descendent_of(Some((hash, parent_hash)));
let is_descendent_of =
descendent_of_builder.build_is_descendent_of(Some((hash, parent_hash)));
let header = PersistedEpochHeader::<E>::from(&epoch.0);
let res = self.inner.import(
hash,
number,
header,
&is_descendent_of,
);
let res = self.inner.import(hash, number, header, &is_descendent_of);
match res {
Ok(_) | Err(fork_tree::Error::Duplicate) => {
@@ -653,8 +626,7 @@ pub type SharedEpochChanges<Block, Epoch> =
#[cfg(test)]
mod tests {
use super::*;
use super::Epoch as EpochT;
use super::{Epoch as EpochT, *};
#[derive(Debug, PartialEq)]
pub struct TestError;
@@ -667,15 +639,14 @@ mod tests {
impl std::error::Error for TestError {}
impl<'a, F: 'a , H: 'a + PartialEq + std::fmt::Debug> IsDescendentOfBuilder<H> for &'a F
where F: Fn(&H, &H) -> Result<bool, TestError>
impl<'a, F: 'a, H: 'a + PartialEq + std::fmt::Debug> IsDescendentOfBuilder<H> for &'a F
where
F: Fn(&H, &H) -> Result<bool, TestError>,
{
type Error = TestError;
type IsDescendentOf = Box<dyn Fn(&H, &H) -> Result<bool, TestError> + 'a>;
fn build_is_descendent_of(&self, current: Option<(H, H)>)
-> Self::IsDescendentOf
{
fn build_is_descendent_of(&self, current: Option<(H, H)>) -> Self::IsDescendentOf {
let f = *self;
Box::new(move |base, head| {
let mut head = head;
@@ -683,7 +654,7 @@ mod tests {
if let Some((ref c_head, ref c_parent)) = current {
if head == c_head {
if base == c_parent {
return Ok(true);
return Ok(true)
} else {
head = c_parent;
}
@@ -709,10 +680,7 @@ mod tests {
type Slot = Slot;
fn increment(&self, _: ()) -> Self {
Epoch {
start_slot: self.start_slot + self.duration,
duration: self.duration,
}
Epoch { start_slot: self.start_slot + self.duration, duration: self.duration }
}
fn end_slot(&self) -> Slot {
@@ -726,7 +694,6 @@ mod tests {
#[test]
fn genesis_epoch_is_created_but_not_imported() {
//
// A - B
// \
// — C
@@ -741,12 +708,10 @@ mod tests {
};
let epoch_changes = EpochChanges::<_, _, Epoch>::new();
let genesis_epoch = epoch_changes.epoch_descriptor_for_child_of(
&is_descendent_of,
b"0",
0,
10101,
).unwrap().unwrap();
let genesis_epoch = epoch_changes
.epoch_descriptor_for_child_of(&is_descendent_of, b"0", 0, 10101)
.unwrap()
.unwrap();
match genesis_epoch {
ViableEpochDescriptor::UnimportedGenesis(slot) => {
@@ -755,12 +720,10 @@ mod tests {
_ => panic!("should be unimported genesis"),
};
let genesis_epoch_2 = epoch_changes.epoch_descriptor_for_child_of(
&is_descendent_of,
b"0",
0,
10102,
).unwrap().unwrap();
let genesis_epoch_2 = epoch_changes
.epoch_descriptor_for_child_of(&is_descendent_of, b"0", 0, 10102)
.unwrap()
.unwrap();
match genesis_epoch_2 {
ViableEpochDescriptor::UnimportedGenesis(slot) => {
@@ -772,7 +735,6 @@ mod tests {
#[test]
fn epoch_changes_between_blocks() {
//
// A - B
// \
// — C
@@ -786,34 +748,23 @@ mod tests {
}
};
let make_genesis = |slot| Epoch {
start_slot: slot,
duration: 100,
};
let make_genesis = |slot| Epoch { start_slot: slot, duration: 100 };
let mut epoch_changes = EpochChanges::<_, _, Epoch>::new();
let genesis_epoch = epoch_changes.epoch_descriptor_for_child_of(
&is_descendent_of,
b"0",
0,
100,
).unwrap().unwrap();
let genesis_epoch = epoch_changes
.epoch_descriptor_for_child_of(&is_descendent_of, b"0", 0, 100)
.unwrap()
.unwrap();
assert_eq!(genesis_epoch, ViableEpochDescriptor::UnimportedGenesis(100));
let import_epoch_1 = epoch_changes
.viable_epoch(&genesis_epoch, &make_genesis)
.unwrap()
.increment(());
let import_epoch_1 =
epoch_changes.viable_epoch(&genesis_epoch, &make_genesis).unwrap().increment(());
let epoch_1 = import_epoch_1.as_ref().clone();
epoch_changes.import(
&is_descendent_of,
*b"A",
1,
*b"0",
import_epoch_1,
).unwrap();
epoch_changes
.import(&is_descendent_of, *b"A", 1, *b"0", import_epoch_1)
.unwrap();
let genesis_epoch = epoch_changes.epoch_data(&genesis_epoch, &make_genesis).unwrap();
assert!(is_descendent_of(b"0", b"A").unwrap());
@@ -823,13 +774,10 @@ mod tests {
{
// x is still within the genesis epoch.
let x = epoch_changes.epoch_data_for_child_of(
&is_descendent_of,
b"A",
1,
end_slot - 1,
&make_genesis,
).unwrap().unwrap();
let x = epoch_changes
.epoch_data_for_child_of(&is_descendent_of, b"A", 1, end_slot - 1, &make_genesis)
.unwrap()
.unwrap();
assert_eq!(x, genesis_epoch);
}
@@ -837,13 +785,10 @@ mod tests {
{
// x is now at the next epoch, because the block is now at the
// start slot of epoch 1.
let x = epoch_changes.epoch_data_for_child_of(
&is_descendent_of,
b"A",
1,
end_slot,
&make_genesis,
).unwrap().unwrap();
let x = epoch_changes
.epoch_data_for_child_of(&is_descendent_of, b"A", 1, end_slot, &make_genesis)
.unwrap()
.unwrap();
assert_eq!(x, epoch_1);
}
@@ -851,13 +796,16 @@ mod tests {
{
// x is now at the next epoch, because the block is now after
// start slot of epoch 1.
let x = epoch_changes.epoch_data_for_child_of(
&is_descendent_of,
b"A",
1,
epoch_1.end_slot() - 1,
&make_genesis,
).unwrap().unwrap();
let x = epoch_changes
.epoch_data_for_child_of(
&is_descendent_of,
b"A",
1,
epoch_1.end_slot() - 1,
&make_genesis,
)
.unwrap()
.unwrap();
assert_eq!(x, epoch_1);
}
@@ -880,90 +828,65 @@ mod tests {
let duration = 100;
let make_genesis = |slot| Epoch {
start_slot: slot,
duration,
};
let make_genesis = |slot| Epoch { start_slot: slot, duration };
let mut epoch_changes = EpochChanges::new();
let next_descriptor = ();
// insert genesis epoch for A
{
let genesis_epoch_a_descriptor = epoch_changes.epoch_descriptor_for_child_of(
&is_descendent_of,
b"0",
0,
100,
).unwrap().unwrap();
let genesis_epoch_a_descriptor = epoch_changes
.epoch_descriptor_for_child_of(&is_descendent_of, b"0", 0, 100)
.unwrap()
.unwrap();
let incremented_epoch = epoch_changes
.viable_epoch(&genesis_epoch_a_descriptor, &make_genesis)
.unwrap()
.increment(next_descriptor.clone());
epoch_changes.import(
&is_descendent_of,
*b"A",
1,
*b"0",
incremented_epoch,
).unwrap();
epoch_changes
.import(&is_descendent_of, *b"A", 1, *b"0", incremented_epoch)
.unwrap();
}
// insert genesis epoch for X
{
let genesis_epoch_x_descriptor = epoch_changes.epoch_descriptor_for_child_of(
&is_descendent_of,
b"0",
0,
1000,
).unwrap().unwrap();
let genesis_epoch_x_descriptor = epoch_changes
.epoch_descriptor_for_child_of(&is_descendent_of, b"0", 0, 1000)
.unwrap()
.unwrap();
let incremented_epoch = epoch_changes
.viable_epoch(&genesis_epoch_x_descriptor, &make_genesis)
.unwrap()
.increment(next_descriptor.clone());
epoch_changes.import(
&is_descendent_of,
*b"X",
1,
*b"0",
incremented_epoch,
).unwrap();
epoch_changes
.import(&is_descendent_of, *b"X", 1, *b"0", incremented_epoch)
.unwrap();
}
// now check that the genesis epochs for our respective block 1s
// respect the chain structure.
{
let epoch_for_a_child = epoch_changes.epoch_data_for_child_of(
&is_descendent_of,
b"A",
1,
101,
&make_genesis,
).unwrap().unwrap();
let epoch_for_a_child = epoch_changes
.epoch_data_for_child_of(&is_descendent_of, b"A", 1, 101, &make_genesis)
.unwrap()
.unwrap();
assert_eq!(epoch_for_a_child, make_genesis(100));
let epoch_for_x_child = epoch_changes.epoch_data_for_child_of(
&is_descendent_of,
b"X",
1,
1001,
&make_genesis,
).unwrap().unwrap();
let epoch_for_x_child = epoch_changes
.epoch_data_for_child_of(&is_descendent_of, b"X", 1, 1001, &make_genesis)
.unwrap()
.unwrap();
assert_eq!(epoch_for_x_child, make_genesis(1000));
let epoch_for_x_child_before_genesis = epoch_changes.epoch_data_for_child_of(
&is_descendent_of,
b"X",
1,
101,
&make_genesis,
).unwrap();
let epoch_for_x_child_before_genesis = epoch_changes
.epoch_data_for_child_of(&is_descendent_of, b"X", 1, 101, &make_genesis)
.unwrap();
// even though there is a genesis epoch at that slot, it's not in
// this chain.
@@ -18,11 +18,11 @@
//! Migration types for epoch changes.
use std::collections::BTreeMap;
use codec::{Encode, Decode};
use crate::{Epoch, EpochChanges, PersistedEpoch, PersistedEpochHeader};
use codec::{Decode, Encode};
use fork_tree::ForkTree;
use sp_runtime::traits::{Block as BlockT, NumberFor};
use crate::{Epoch, EpochChanges, PersistedEpoch, PersistedEpochHeader};
use std::collections::BTreeMap;
/// Legacy definition of epoch changes.
#[derive(Clone, Encode, Decode)]
@@ -31,9 +31,11 @@ pub struct EpochChangesV0<Hash, Number, E: Epoch> {
}
/// Type alias for legacy definition of epoch changes.
pub type EpochChangesForV0<Block, Epoch> = EpochChangesV0<<Block as BlockT>::Hash, NumberFor<Block>, Epoch>;
pub type EpochChangesForV0<Block, Epoch> =
EpochChangesV0<<Block as BlockT>::Hash, NumberFor<Block>, Epoch>;
impl<Hash, Number, E: Epoch> EpochChangesV0<Hash, Number, E> where
impl<Hash, Number, E: Epoch> EpochChangesV0<Hash, Number, E>
where
Hash: PartialEq + Ord + Copy,
Number: Ord + Copy,
{
@@ -19,26 +19,30 @@
//! Extensions for manual seal to produce blocks valid for any runtime.
use super::Error;
use sp_runtime::traits::{Block as BlockT, DigestFor};
use sp_inherents::InherentData;
use sp_consensus::BlockImportParams;
use sp_inherents::InherentData;
use sp_runtime::traits::{Block as BlockT, DigestFor};
pub mod babe;
/// Consensus data provider, manual seal uses this trait object for authoring blocks valid
/// Consensus data provider, manual seal uses this trait object for authoring blocks valid
/// for any runtime.
pub trait ConsensusDataProvider<B: BlockT>: Send + Sync {
/// Block import transaction type
type Transaction;
/// Attempt to create a consensus digest.
fn create_digest(&self, parent: &B::Header, inherents: &InherentData) -> Result<DigestFor<B>, Error>;
fn create_digest(
&self,
parent: &B::Header,
inherents: &InherentData,
) -> Result<DigestFor<B>, Error>;
/// set up the neccessary import params.
fn append_block_import(
&self,
parent: &B::Header,
params: &mut BlockImportParams<B, Self::Transaction>,
inherents: &InherentData
inherents: &InherentData,
) -> Result<(), Error>;
}
@@ -21,30 +21,40 @@
use super::ConsensusDataProvider;
use crate::Error;
use codec::Encode;
use std::{borrow::Cow, sync::{Arc, atomic}, time::SystemTime};
use sc_client_api::{AuxStore, UsageProvider};
use sc_consensus_babe::{
Config, Epoch, authorship, CompatibleDigestItem, BabeIntermediate, INTERMEDIATE_KEY,
find_pre_digest,
authorship, find_pre_digest, BabeIntermediate, CompatibleDigestItem, Config, Epoch,
INTERMEDIATE_KEY,
};
use sc_consensus_epochs::{
descendent_query, EpochHeader, SharedEpochChanges, ViableEpochDescriptor,
};
use sc_consensus_epochs::{SharedEpochChanges, descendent_query, ViableEpochDescriptor, EpochHeader};
use sp_keystore::SyncCryptoStorePtr;
use std::{
borrow::Cow,
sync::{atomic, Arc},
time::SystemTime,
};
use sp_api::{ProvideRuntimeApi, TransactionFor};
use sp_blockchain::{HeaderBackend, HeaderMetadata};
use sp_consensus::{BlockImportParams, BlockOrigin, ForkChoiceStrategy};
use sp_consensus_slots::Slot;
use sp_consensus_babe::{
BabeApi, inherents::BabeInherentData, ConsensusLog, BABE_ENGINE_ID, AuthorityId,
digests::{PreDigest, SecondaryPlainPreDigest, NextEpochDescriptor}, BabeAuthorityWeight,
use sp_consensus::{
import_queue::{CacheKeyId, Verifier},
BlockImportParams, BlockOrigin, ForkChoiceStrategy,
};
use sp_consensus_babe::{
digests::{NextEpochDescriptor, PreDigest, SecondaryPlainPreDigest},
inherents::BabeInherentData,
AuthorityId, BabeApi, BabeAuthorityWeight, ConsensusLog, BABE_ENGINE_ID,
};
use sp_consensus_slots::Slot;
use sp_inherents::{InherentData, InherentDataProvider, InherentIdentifier};
use sp_runtime::{
traits::{DigestItemFor, DigestFor, Block as BlockT, Zero, Header},
generic::{Digest, BlockId}, Justifications,
generic::{BlockId, Digest},
traits::{Block as BlockT, DigestFor, DigestItemFor, Header, Zero},
Justifications,
};
use sp_timestamp::{InherentType, INHERENT_IDENTIFIER, TimestampInherentData};
use sp_consensus::import_queue::{Verifier, CacheKeyId};
use sp_timestamp::{InherentType, TimestampInherentData, INHERENT_IDENTIFIER};
/// Provides BABE-compatible predigests and BlockImportParams.
/// Intended for use with BABE runtimes.
@@ -77,19 +87,16 @@ pub struct BabeVerifier<B: BlockT, C> {
impl<B: BlockT, C> BabeVerifier<B, C> {
/// create a nrew verifier
pub fn new(epoch_changes: SharedEpochChanges<B, Epoch>, client: Arc<C>) -> BabeVerifier<B, C> {
BabeVerifier {
epoch_changes,
client,
}
BabeVerifier { epoch_changes, client }
}
}
/// The verifier for the manual seal engine; instantly finalizes.
#[async_trait::async_trait]
impl<B, C> Verifier<B> for BabeVerifier<B, C>
where
B: BlockT,
C: HeaderBackend<B> + HeaderMetadata<B, Error = sp_blockchain::Error>
where
B: BlockT,
C: HeaderBackend<B> + HeaderMetadata<B, Error = sp_blockchain::Error>,
{
async fn verify(
&mut self,
@@ -107,7 +114,9 @@ impl<B, C> Verifier<B> for BabeVerifier<B, C>
let pre_digest = find_pre_digest::<B>(&header)?;
let parent_hash = header.parent_hash();
let parent = self.client.header(BlockId::Hash(*parent_hash))
let parent = self
.client
.header(BlockId::Hash(*parent_hash))
.ok()
.flatten()
.ok_or_else(|| format!("header for block {} not found", parent_hash))?;
@@ -134,14 +143,14 @@ impl<B, C> Verifier<B> for BabeVerifier<B, C>
}
impl<B, C> BabeConsensusDataProvider<B, C>
where
B: BlockT,
C: AuxStore
+ HeaderBackend<B>
+ ProvideRuntimeApi<B>
+ HeaderMetadata<B, Error = sp_blockchain::Error>
+ UsageProvider<B>,
C::Api: BabeApi<B>,
where
B: BlockT,
C: AuxStore
+ HeaderBackend<B>
+ ProvideRuntimeApi<B>
+ HeaderMetadata<B, Error = sp_blockchain::Error>
+ UsageProvider<B>,
C::Api: BabeApi<B>,
{
pub fn new(
client: Arc<C>,
@@ -155,13 +164,7 @@ impl<B, C> BabeConsensusDataProvider<B, C>
let config = Config::get_or_compute(&*client)?;
Ok(Self {
config,
client,
keystore,
epoch_changes,
authorities,
})
Ok(Self { config, client, keystore, epoch_changes, authorities })
}
fn epoch(&self, parent: &B::Header, slot: Slot) -> Result<Epoch, Error> {
@@ -177,10 +180,7 @@ impl<B, C> BabeConsensusDataProvider<B, C>
.ok_or_else(|| sp_consensus::Error::InvalidAuthoritiesSet)?;
let epoch = epoch_changes
.viable_epoch(
&epoch_descriptor,
|slot| Epoch::genesis(&self.config, slot),
)
.viable_epoch(&epoch_descriptor, |slot| Epoch::genesis(&self.config, slot))
.ok_or_else(|| {
log::info!(target: "babe", "create_digest: no viable_epoch :(");
sp_consensus::Error::InvalidAuthoritiesSet
@@ -191,38 +191,37 @@ impl<B, C> BabeConsensusDataProvider<B, C>
}
impl<B, C> ConsensusDataProvider<B> for BabeConsensusDataProvider<B, C>
where
B: BlockT,
C: AuxStore
+ HeaderBackend<B>
+ HeaderMetadata<B, Error = sp_blockchain::Error>
+ UsageProvider<B>
+ ProvideRuntimeApi<B>,
C::Api: BabeApi<B>,
where
B: BlockT,
C: AuxStore
+ HeaderBackend<B>
+ HeaderMetadata<B, Error = sp_blockchain::Error>
+ UsageProvider<B>
+ ProvideRuntimeApi<B>,
C::Api: BabeApi<B>,
{
type Transaction = TransactionFor<C, B>;
fn create_digest(&self, parent: &B::Header, inherents: &InherentData) -> Result<DigestFor<B>, Error> {
let slot = inherents.babe_inherent_data()?
fn create_digest(
&self,
parent: &B::Header,
inherents: &InherentData,
) -> Result<DigestFor<B>, Error> {
let slot = inherents
.babe_inherent_data()?
.ok_or_else(|| Error::StringError("No babe inherent data".into()))?;
let epoch = self.epoch(parent, slot)?;
// this is a dev node environment, we should always be able to claim a slot.
let logs = if let Some((predigest, _)) = authorship::claim_slot(
slot,
&epoch,
&self.keystore,
) {
vec![
<DigestItemFor<B> as CompatibleDigestItem>::babe_pre_digest(predigest),
]
let logs = if let Some((predigest, _)) =
authorship::claim_slot(slot, &epoch, &self.keystore)
{
vec![<DigestItemFor<B> as CompatibleDigestItem>::babe_pre_digest(predigest)]
} else {
// well we couldn't claim a slot because this is an existing chain and we're not in the authorities.
// we need to tell BabeBlockImport that the epoch has changed, and we put ourselves in the authorities.
let predigest = PreDigest::SecondaryPlain(SecondaryPlainPreDigest {
slot,
authority_index: 0_u32,
});
let predigest =
PreDigest::SecondaryPlain(SecondaryPlainPreDigest { slot, authority_index: 0_u32 });
let mut epoch_changes = self.epoch_changes.shared_data();
let epoch_descriptor = epoch_changes
@@ -232,12 +231,15 @@ impl<B, C> ConsensusDataProvider<B> for BabeConsensusDataProvider<B, C>
parent.number().clone(),
slot,
)
.map_err(|e| Error::StringError(format!("failed to fetch epoch_descriptor: {}", e)))?
.map_err(|e| {
Error::StringError(format!("failed to fetch epoch_descriptor: {}", e))
})?
.ok_or_else(|| sp_consensus::Error::InvalidAuthoritiesSet)?;
match epoch_descriptor {
ViableEpochDescriptor::Signaled(identifier, _epoch_header) => {
let epoch_mut = epoch_changes.epoch_mut(&identifier)
let epoch_mut = epoch_changes
.epoch_mut(&identifier)
.ok_or_else(|| sp_consensus::Error::InvalidAuthoritiesSet)?;
// mutate the current epoch
@@ -251,15 +253,13 @@ impl<B, C> ConsensusDataProvider<B> for BabeConsensusDataProvider<B, C>
vec![
DigestItemFor::<B>::PreRuntime(BABE_ENGINE_ID, predigest.encode()),
DigestItemFor::<B>::Consensus(BABE_ENGINE_ID, next_epoch.encode())
DigestItemFor::<B>::Consensus(BABE_ENGINE_ID, next_epoch.encode()),
]
},
ViableEpochDescriptor::UnimportedGenesis(_) => {
// since this is the genesis, secondary predigest works for now.
vec![
DigestItemFor::<B>::PreRuntime(BABE_ENGINE_ID, predigest.encode()),
]
}
vec![DigestItemFor::<B>::PreRuntime(BABE_ENGINE_ID, predigest.encode())]
},
}
};
@@ -270,9 +270,10 @@ impl<B, C> ConsensusDataProvider<B> for BabeConsensusDataProvider<B, C>
&self,
parent: &B::Header,
params: &mut BlockImportParams<B, Self::Transaction>,
inherents: &InherentData
inherents: &InherentData,
) -> Result<(), Error> {
let slot = inherents.babe_inherent_data()?
let slot = inherents
.babe_inherent_data()?
.ok_or_else(|| Error::StringError("No babe inherent data".into()))?;
let epoch_changes = self.epoch_changes.shared_data();
let mut epoch_descriptor = epoch_changes
@@ -289,27 +290,27 @@ impl<B, C> ConsensusDataProvider<B> for BabeConsensusDataProvider<B, C>
// a quick check to see if we're in the authorities
let epoch = self.epoch(parent, slot)?;
let (authority, _) = self.authorities.first().expect("authorities is non-emptyp; qed");
let has_authority = epoch.authorities.iter()
.find(|(id, _)| *id == *authority)
.is_some();
let has_authority = epoch.authorities.iter().find(|(id, _)| *id == *authority).is_some();
if !has_authority {
log::info!(target: "manual-seal", "authority not found");
let timestamp = inherents.timestamp_inherent_data()?
let timestamp = inherents
.timestamp_inherent_data()?
.ok_or_else(|| Error::StringError("No timestamp inherent data".into()))?;
let slot = *timestamp / self.config.slot_duration;
// manually hard code epoch descriptor
epoch_descriptor = match epoch_descriptor {
ViableEpochDescriptor::Signaled(identifier, _header) => {
ViableEpochDescriptor::Signaled(identifier, _header) =>
ViableEpochDescriptor::Signaled(
identifier,
EpochHeader {
start_slot: slot.into(),
end_slot: (slot * self.config.epoch_length).into(),
},
)
},
_ => unreachable!("we're not in the authorities, so this isn't the genesis epoch; qed")
),
_ => unreachable!(
"we're not in the authorities, so this isn't the genesis epoch; qed"
),
};
}
@@ -326,16 +327,16 @@ impl<B, C> ConsensusDataProvider<B> for BabeConsensusDataProvider<B, C>
/// Mocks the timestamp inherent to always produce the timestamp for the next babe slot.
pub struct SlotTimestampProvider {
time: atomic::AtomicU64,
slot_duration: u64
slot_duration: u64,
}
impl SlotTimestampProvider {
/// Create a new mocked time stamp provider.
pub fn new<B, C>(client: Arc<C>) -> Result<Self, Error>
where
B: BlockT,
C: AuxStore + HeaderBackend<B> + ProvideRuntimeApi<B> + UsageProvider<B>,
C::Api: BabeApi<B>,
where
B: BlockT,
C: AuxStore + HeaderBackend<B> + ProvideRuntimeApi<B> + UsageProvider<B>,
C::Api: BabeApi<B>,
{
let slot_duration = Config::get_or_compute(&*client)?.slot_duration;
let info = client.info();
@@ -355,10 +356,7 @@ impl SlotTimestampProvider {
.as_millis() as u64
};
Ok(Self {
time: atomic::AtomicU64::new(time),
slot_duration,
})
Ok(Self { time: atomic::AtomicU64::new(time), slot_duration })
}
/// Get the current slot number
@@ -369,12 +367,13 @@ impl SlotTimestampProvider {
#[async_trait::async_trait]
impl InherentDataProvider for SlotTimestampProvider {
fn provide_inherent_data(&self, inherent_data: &mut InherentData) -> Result<(), sp_inherents::Error> {
fn provide_inherent_data(
&self,
inherent_data: &mut InherentData,
) -> Result<(), sp_inherents::Error> {
// we update the time here.
let duration: InherentType = self.time.fetch_add(
self.slot_duration,
atomic::Ordering::SeqCst,
).into();
let duration: InherentType =
self.time.fetch_add(self.slot_duration, atomic::Ordering::SeqCst).into();
inherent_data.put_data(INHERENT_IDENTIFIER, &duration)?;
Ok(())
}
@@ -19,10 +19,10 @@
//! A manual sealing engine: the engine listens for rpc calls to seal blocks and create forks.
//! This is suitable for a testing environment.
use sp_consensus::{Error as ConsensusError, ImportResult};
use futures::channel::{mpsc::SendError, oneshot};
use sp_blockchain::Error as BlockchainError;
use sp_consensus::{Error as ConsensusError, ImportResult};
use sp_inherents::Error as InherentsError;
use futures::channel::{oneshot, mpsc::SendError};
/// Error code for rpc
mod codes {
@@ -63,14 +63,14 @@ pub enum Error {
#[display(fmt = "{}", _0)]
#[from(ignore)]
StringError(String),
///send error
/// send error
#[display(fmt = "Consensus process is terminating")]
Canceled(oneshot::Canceled),
///send error
/// send error
#[display(fmt = "Consensus process is terminating")]
SendError(SendError),
/// Some other error.
#[display(fmt="Other error: {}", _0)]
#[display(fmt = "Other error: {}", _0)]
Other(Box<dyn std::error::Error + Send>),
}
@@ -85,7 +85,7 @@ impl Error {
InherentError(_) => codes::INHERENTS_ERROR,
BlockchainError(_) => codes::BLOCKCHAIN_ERROR,
SendError(_) | Canceled(_) => codes::SERVER_SHUTTING_DOWN,
_ => codes::UNKNOWN_ERROR
_ => codes::UNKNOWN_ERROR,
}
}
}
@@ -95,7 +95,7 @@ impl std::convert::From<Error> for jsonrpc_core::Error {
jsonrpc_core::Error {
code: jsonrpc_core::ErrorCode::ServerError(error.to_code()),
message: format!("{}", error),
data: None
data: None,
}
}
}
@@ -19,14 +19,9 @@
//! Block finalization utilities
use crate::rpc;
use sp_runtime::{
Justification,
traits::Block as BlockT,
generic::BlockId,
};
use std::sync::Arc;
use sc_client_api::backend::{Backend as ClientBackend, Finalizer};
use std::marker::PhantomData;
use sp_runtime::{generic::BlockId, traits::Block as BlockT, Justification};
use std::{marker::PhantomData, sync::Arc};
/// params for block finalization.
pub struct FinalizeBlockParams<B: BlockT, F, CB> {
@@ -42,30 +37,23 @@ pub struct FinalizeBlockParams<B: BlockT, F, CB> {
pub _phantom: PhantomData<CB>,
}
/// finalizes a block in the backend with the given params.
pub async fn finalize_block<B, F, CB>(params: FinalizeBlockParams<B, F, CB>)
where
B: BlockT,
F: Finalizer<B, CB>,
CB: ClientBackend<B>,
where
B: BlockT,
F: Finalizer<B, CB>,
CB: ClientBackend<B>,
{
let FinalizeBlockParams {
hash,
mut sender,
justification,
finalizer,
..
} = params;
let FinalizeBlockParams { hash, mut sender, justification, finalizer, .. } = params;
match finalizer.finalize_block(BlockId::Hash(hash), justification, true) {
Err(e) => {
log::warn!("Failed to finalize block {:?}", e);
rpc::send_result(&mut sender, Err(e.into()))
}
},
Ok(()) => {
log::info!("✅ Successfully finalized block: {}", hash);
rpc::send_result(&mut sender, Ok(()))
}
},
}
}
+171 -201
View File
@@ -20,17 +20,17 @@
//! This is suitable for a testing environment.
use futures::prelude::*;
use sp_consensus::{
Environment, Proposer, SelectChain, BlockImport,
ForkChoiceStrategy, BlockImportParams, BlockOrigin,
import_queue::{Verifier, BasicQueue, CacheKeyId, BoxBlockImport},
};
use sp_blockchain::HeaderBackend;
use sp_inherents::CreateInherentDataProviders;
use sp_runtime::{traits::Block as BlockT, Justifications, ConsensusEngineId};
use sc_client_api::backend::{Backend as ClientBackend, Finalizer};
use std::{sync::Arc, marker::PhantomData};
use prometheus_endpoint::Registry;
use sc_client_api::backend::{Backend as ClientBackend, Finalizer};
use sp_blockchain::HeaderBackend;
use sp_consensus::{
import_queue::{BasicQueue, BoxBlockImport, CacheKeyId, Verifier},
BlockImport, BlockImportParams, BlockOrigin, Environment, ForkChoiceStrategy, Proposer,
SelectChain,
};
use sp_inherents::CreateInherentDataProviders;
use sp_runtime::{traits::Block as BlockT, ConsensusEngineId, Justifications};
use std::{marker::PhantomData, sync::Arc};
mod error;
mod finalize_block;
@@ -40,14 +40,14 @@ pub mod consensus;
pub mod rpc;
pub use self::{
error::Error,
consensus::ConsensusDataProvider,
error::Error,
finalize_block::{finalize_block, FinalizeBlockParams},
seal_block::{SealBlockParams, seal_block, MAX_PROPOSAL_DURATION},
rpc::{EngineCommand, CreatedBlock},
rpc::{CreatedBlock, EngineCommand},
seal_block::{seal_block, SealBlockParams, MAX_PROPOSAL_DURATION},
};
use sp_api::{ProvideRuntimeApi, TransactionFor};
use sc_transaction_pool_api::TransactionPool;
use sp_api::{ProvideRuntimeApi, TransactionFor};
/// The `ConsensusEngineId` of Manual Seal.
pub const MANUAL_SEAL_ENGINE_ID: ConsensusEngineId = [b'm', b'a', b'n', b'l'];
@@ -80,17 +80,11 @@ pub fn import_queue<Block, Transaction>(
spawner: &impl sp_core::traits::SpawnEssentialNamed,
registry: Option<&Registry>,
) -> BasicQueue<Block, Transaction>
where
Block: BlockT,
Transaction: Send + Sync + 'static,
where
Block: BlockT,
Transaction: Send + Sync + 'static,
{
BasicQueue::new(
ManualSealVerifier,
block_import,
None,
spawner,
registry,
)
BasicQueue::new(ManualSealVerifier, block_import, None, spawner, registry)
}
/// Params required to start the instant sealing authorship task.
@@ -115,7 +109,8 @@ pub struct ManualSealParams<B: BlockT, BI, E, C: ProvideRuntimeApi<B>, TP, SC, C
pub select_chain: SC,
/// Digest provider for inclusion in blocks.
pub consensus_data_provider: Option<Box<dyn ConsensusDataProvider<B, Transaction = TransactionFor<C, B>>>>,
pub consensus_data_provider:
Option<Box<dyn ConsensusDataProvider<B, Transaction = TransactionFor<C, B>>>>,
/// Something that can create the inherent data providers.
pub create_inherent_data_providers: CIDP,
@@ -139,7 +134,8 @@ pub struct InstantSealParams<B: BlockT, BI, E, C: ProvideRuntimeApi<B>, TP, SC,
pub select_chain: SC,
/// Digest provider for inclusion in blocks.
pub consensus_data_provider: Option<Box<dyn ConsensusDataProvider<B, Transaction = TransactionFor<C, B>>>>,
pub consensus_data_provider:
Option<Box<dyn ConsensusDataProvider<B, Transaction = TransactionFor<C, B>>>>,
/// Something that can create the inherent data providers.
pub create_inherent_data_providers: CIDP,
@@ -156,58 +152,52 @@ pub async fn run_manual_seal<B, BI, CB, E, C, TP, SC, CS, CIDP>(
select_chain,
consensus_data_provider,
create_inherent_data_providers,
}: ManualSealParams<B, BI, E, C, TP, SC, CS, CIDP>
)
where
B: BlockT + 'static,
BI: BlockImport<B, Error = sp_consensus::Error, Transaction = sp_api::TransactionFor<C, B>>
+ Send + Sync + 'static,
C: HeaderBackend<B> + Finalizer<B, CB> + ProvideRuntimeApi<B> + 'static,
CB: ClientBackend<B> + 'static,
E: Environment<B> + 'static,
E::Proposer: Proposer<B, Transaction = TransactionFor<C, B>>,
CS: Stream<Item=EngineCommand<<B as BlockT>::Hash>> + Unpin + 'static,
SC: SelectChain<B> + 'static,
TransactionFor<C, B>: 'static,
TP: TransactionPool<Block = B>,
CIDP: CreateInherentDataProviders<B, ()>,
}: ManualSealParams<B, BI, E, C, TP, SC, CS, CIDP>,
) where
B: BlockT + 'static,
BI: BlockImport<B, Error = sp_consensus::Error, Transaction = sp_api::TransactionFor<C, B>>
+ Send
+ Sync
+ 'static,
C: HeaderBackend<B> + Finalizer<B, CB> + ProvideRuntimeApi<B> + 'static,
CB: ClientBackend<B> + 'static,
E: Environment<B> + 'static,
E::Proposer: Proposer<B, Transaction = TransactionFor<C, B>>,
CS: Stream<Item = EngineCommand<<B as BlockT>::Hash>> + Unpin + 'static,
SC: SelectChain<B> + 'static,
TransactionFor<C, B>: 'static,
TP: TransactionPool<Block = B>,
CIDP: CreateInherentDataProviders<B, ()>,
{
while let Some(command) = commands_stream.next().await {
match command {
EngineCommand::SealNewBlock {
create_empty,
finalize,
parent_hash,
sender,
} => {
seal_block(
SealBlockParams {
sender,
parent_hash,
finalize,
create_empty,
env: &mut env,
select_chain: &select_chain,
block_import: &mut block_import,
consensus_data_provider: consensus_data_provider.as_ref().map(|p| &**p),
pool: pool.clone(),
client: client.clone(),
create_inherent_data_providers: &create_inherent_data_providers,
}
).await;
}
EngineCommand::SealNewBlock { create_empty, finalize, parent_hash, sender } => {
seal_block(SealBlockParams {
sender,
parent_hash,
finalize,
create_empty,
env: &mut env,
select_chain: &select_chain,
block_import: &mut block_import,
consensus_data_provider: consensus_data_provider.as_ref().map(|p| &**p),
pool: pool.clone(),
client: client.clone(),
create_inherent_data_providers: &create_inherent_data_providers,
})
.await;
},
EngineCommand::FinalizeBlock { hash, sender, justification } => {
let justification = justification.map(|j| (MANUAL_SEAL_ENGINE_ID, j));
finalize_block(
FinalizeBlockParams {
hash,
sender,
justification,
finalizer: client.clone(),
_phantom: PhantomData,
}
).await
}
finalize_block(FinalizeBlockParams {
hash,
sender,
justification,
finalizer: client.clone(),
_phantom: PhantomData,
})
.await
},
}
}
}
@@ -224,63 +214,57 @@ pub async fn run_instant_seal<B, BI, CB, E, C, TP, SC, CIDP>(
select_chain,
consensus_data_provider,
create_inherent_data_providers,
}: InstantSealParams<B, BI, E, C, TP, SC, CIDP>
)
where
B: BlockT + 'static,
BI: BlockImport<B, Error = sp_consensus::Error, Transaction = sp_api::TransactionFor<C, B>>
+ Send + Sync + 'static,
C: HeaderBackend<B> + Finalizer<B, CB> + ProvideRuntimeApi<B> + 'static,
CB: ClientBackend<B> + 'static,
E: Environment<B> + 'static,
E::Proposer: Proposer<B, Transaction = TransactionFor<C, B>>,
SC: SelectChain<B> + 'static,
TransactionFor<C, B>: 'static,
TP: TransactionPool<Block = B>,
CIDP: CreateInherentDataProviders<B, ()>,
}: InstantSealParams<B, BI, E, C, TP, SC, CIDP>,
) where
B: BlockT + 'static,
BI: BlockImport<B, Error = sp_consensus::Error, Transaction = sp_api::TransactionFor<C, B>>
+ Send
+ Sync
+ 'static,
C: HeaderBackend<B> + Finalizer<B, CB> + ProvideRuntimeApi<B> + 'static,
CB: ClientBackend<B> + 'static,
E: Environment<B> + 'static,
E::Proposer: Proposer<B, Transaction = TransactionFor<C, B>>,
SC: SelectChain<B> + 'static,
TransactionFor<C, B>: 'static,
TP: TransactionPool<Block = B>,
CIDP: CreateInherentDataProviders<B, ()>,
{
// instant-seal creates blocks as soon as transactions are imported
// into the transaction pool.
let commands_stream = pool.import_notification_stream()
.map(|_| {
EngineCommand::SealNewBlock {
create_empty: false,
finalize: false,
parent_hash: None,
sender: None,
}
});
let commands_stream = pool.import_notification_stream().map(|_| EngineCommand::SealNewBlock {
create_empty: false,
finalize: false,
parent_hash: None,
sender: None,
});
run_manual_seal(
ManualSealParams {
block_import,
env,
client,
pool,
commands_stream,
select_chain,
consensus_data_provider,
create_inherent_data_providers,
}
).await
run_manual_seal(ManualSealParams {
block_import,
env,
client,
pool,
commands_stream,
select_chain,
consensus_data_provider,
create_inherent_data_providers,
})
.await
}
#[cfg(test)]
mod tests {
use super::*;
use substrate_test_runtime_client::{
DefaultTestClientBuilderExt,
TestClientBuilderExt,
AccountKeyring::*,
TestClientBuilder,
};
use sc_transaction_pool::{BasicPool, RevalidationType, Options};
use substrate_test_runtime_transaction_pool::{TestApi, uxt};
use sc_transaction_pool_api::{TransactionPool, MaintainedTransactionPool, TransactionSource};
use sp_runtime::generic::BlockId;
use sp_consensus::ImportedAux;
use sc_basic_authorship::ProposerFactory;
use sc_client_api::BlockBackend;
use sc_transaction_pool::{BasicPool, Options, RevalidationType};
use sc_transaction_pool_api::{MaintainedTransactionPool, TransactionPool, TransactionSource};
use sp_consensus::ImportedAux;
use sp_runtime::generic::BlockId;
use substrate_test_runtime_client::{
AccountKeyring::*, DefaultTestClientBuilderExt, TestClientBuilder, TestClientBuilderExt,
};
use substrate_test_runtime_transaction_pool::{uxt, TestApi};
fn api() -> Arc<TestApi> {
Arc::new(TestApi::empty())
@@ -303,40 +287,32 @@ mod tests {
spawner.clone(),
0,
));
let env = ProposerFactory::new(
spawner.clone(),
client.clone(),
pool.clone(),
None,
None,
);
let env = ProposerFactory::new(spawner.clone(), client.clone(), pool.clone(), None, None);
// this test checks that blocks are created as soon as transactions are imported into the pool.
let (sender, receiver) = futures::channel::oneshot::channel();
let mut sender = Arc::new(Some(sender));
let commands_stream = pool.pool().validated_pool().import_notification_stream()
.map(move |_| {
let commands_stream =
pool.pool().validated_pool().import_notification_stream().map(move |_| {
// we're only going to submit one tx so this fn will only be called once.
let mut_sender = Arc::get_mut(&mut sender).unwrap();
let mut_sender = Arc::get_mut(&mut sender).unwrap();
let sender = std::mem::take(mut_sender);
EngineCommand::SealNewBlock {
create_empty: false,
finalize: true,
parent_hash: None,
sender
sender,
}
});
let future = run_manual_seal(
ManualSealParams {
block_import: client.clone(),
env,
client: client.clone(),
pool: pool.clone(),
commands_stream,
select_chain,
create_inherent_data_providers: |_, _| async { Ok(()) },
consensus_data_provider: None,
}
);
let future = run_manual_seal(ManualSealParams {
block_import: client.clone(),
env,
client: client.clone(),
pool: pool.clone(),
commands_stream,
select_chain,
create_inherent_data_providers: |_, _| async { Ok(()) },
consensus_data_provider: None,
});
std::thread::spawn(|| {
let mut rt = tokio::runtime::Runtime::new().unwrap();
// spawn the background authorship task
@@ -380,27 +356,19 @@ mod tests {
spawner.clone(),
0,
));
let env = ProposerFactory::new(
spawner.clone(),
client.clone(),
pool.clone(),
None,
None,
);
let env = ProposerFactory::new(spawner.clone(), client.clone(), pool.clone(), None, None);
// this test checks that blocks are created as soon as an engine command is sent over the stream.
let (mut sink, commands_stream) = futures::channel::mpsc::channel(1024);
let future = run_manual_seal(
ManualSealParams {
block_import: client.clone(),
env,
client: client.clone(),
pool: pool.clone(),
commands_stream,
select_chain,
consensus_data_provider: None,
create_inherent_data_providers: |_, _| async { Ok(()) },
}
);
let future = run_manual_seal(ManualSealParams {
block_import: client.clone(),
env,
client: client.clone(),
pool: pool.clone(),
commands_stream,
select_chain,
consensus_data_provider: None,
create_inherent_data_providers: |_, _| async { Ok(()) },
});
std::thread::spawn(|| {
let mut rt = tokio::runtime::Runtime::new().unwrap();
// spawn the background authorship task
@@ -416,7 +384,9 @@ mod tests {
sender: Some(tx),
create_empty: false,
finalize: false,
}).await.unwrap();
})
.await
.unwrap();
let created_block = rx.await.unwrap().unwrap();
// assert that the background task returns ok
@@ -439,8 +409,10 @@ mod tests {
sink.send(EngineCommand::FinalizeBlock {
sender: Some(tx),
hash: header.hash(),
justification: None
}).await.unwrap();
justification: None,
})
.await
.unwrap();
// assert that the background task returns ok
assert_eq!(rx.await.unwrap().unwrap(), ());
}
@@ -461,27 +433,19 @@ mod tests {
spawner.clone(),
0,
));
let env = ProposerFactory::new(
spawner.clone(),
client.clone(),
pool.clone(),
None,
None,
);
let env = ProposerFactory::new(spawner.clone(), client.clone(), pool.clone(), None, None);
// this test checks that blocks are created as soon as an engine command is sent over the stream.
let (mut sink, commands_stream) = futures::channel::mpsc::channel(1024);
let future = run_manual_seal(
ManualSealParams {
block_import: client.clone(),
env,
client: client.clone(),
pool: pool.clone(),
commands_stream,
select_chain,
consensus_data_provider: None,
create_inherent_data_providers: |_, _| async { Ok(()) },
}
);
let future = run_manual_seal(ManualSealParams {
block_import: client.clone(),
env,
client: client.clone(),
pool: pool.clone(),
commands_stream,
select_chain,
consensus_data_provider: None,
create_inherent_data_providers: |_, _| async { Ok(()) },
});
std::thread::spawn(|| {
let mut rt = tokio::runtime::Runtime::new().unwrap();
// spawn the background authorship task
@@ -498,7 +462,9 @@ mod tests {
sender: Some(tx),
create_empty: false,
finalize: false,
}).await.unwrap();
})
.await
.unwrap();
let created_block = rx.await.unwrap().unwrap();
pool_api.increment_nonce(Alice.into());
@@ -524,31 +490,35 @@ mod tests {
pool.maintain(sc_transaction_pool_api::ChainEvent::NewBestBlock {
hash: header.hash(),
tree_route: None,
}).await;
})
.await;
let (tx1, rx1) = futures::channel::oneshot::channel();
assert!(sink.send(EngineCommand::SealNewBlock {
parent_hash: Some(created_block.hash),
sender: Some(tx1),
create_empty: false,
finalize: false,
}).await.is_ok());
assert_matches::assert_matches!(
rx1.await.expect("should be no error receiving"),
Ok(_)
);
assert!(sink
.send(EngineCommand::SealNewBlock {
parent_hash: Some(created_block.hash),
sender: Some(tx1),
create_empty: false,
finalize: false,
})
.await
.is_ok());
assert_matches::assert_matches!(rx1.await.expect("should be no error receiving"), Ok(_));
let block = client.block(&BlockId::Number(2)).unwrap().unwrap().block;
pool_api.add_block(block, true);
pool_api.increment_nonce(Alice.into());
assert!(pool.submit_one(&BlockId::Number(1), SOURCE, uxt(Bob, 0)).await.is_ok());
let (tx2, rx2) = futures::channel::oneshot::channel();
assert!(sink.send(EngineCommand::SealNewBlock {
parent_hash: Some(created_block.hash),
sender: Some(tx2),
create_empty: false,
finalize: false,
}).await.is_ok());
assert!(sink
.send(EngineCommand::SealNewBlock {
parent_hash: Some(created_block.hash),
sender: Some(tx2),
create_empty: false,
finalize: false,
})
.await
.is_ok());
let imported = rx2.await.unwrap().unwrap();
// assert that fork block is in the db
assert!(client.header(&BlockId::Hash(imported.hash)).unwrap().is_some())
@@ -18,18 +18,16 @@
//! RPC interface for the `ManualSeal` Engine.
use sp_consensus::ImportedAux;
use jsonrpc_core::Error;
use jsonrpc_derive::rpc;
pub use self::gen_client::Client as ManualSealClient;
use futures::{
channel::{mpsc, oneshot},
TryFutureExt,
FutureExt,
SinkExt
FutureExt, SinkExt, TryFutureExt,
};
use jsonrpc_core::Error;
use jsonrpc_derive::rpc;
use serde::{Deserialize, Serialize};
use sp_consensus::ImportedAux;
use sp_runtime::EncodedJustification;
pub use self::gen_client::Client as ManualSealClient;
/// Future's type for jsonrpc
type FutureResult<T> = Box<dyn jsonrpc_core::futures::Future<Item = T, Error = Error> + Send>;
@@ -63,7 +61,7 @@ pub enum EngineCommand<Hash> {
sender: Sender<()>,
/// finalization justification
justification: Option<EncodedJustification>,
}
},
}
/// RPC trait that provides methods for interacting with the manual-seal authorship task over rpc.
@@ -75,7 +73,7 @@ pub trait ManualSealApi<Hash> {
&self,
create_empty: bool,
finalize: bool,
parent_hash: Option<Hash>
parent_hash: Option<Hash>,
) -> FutureResult<CreatedBlock<Hash>>;
/// Instructs the manual-seal authorship task to finalize a block
@@ -83,7 +81,7 @@ pub trait ManualSealApi<Hash> {
fn finalize_block(
&self,
hash: Hash,
justification: Option<EncodedJustification>
justification: Option<EncodedJustification>,
) -> FutureResult<bool>;
}
@@ -98,7 +96,7 @@ pub struct CreatedBlock<Hash> {
/// hash of the created block.
pub hash: Hash,
/// some extra details about the import operation
pub aux: ImportedAux
pub aux: ImportedAux,
}
impl<Hash> ManualSeal<Hash> {
@@ -113,7 +111,7 @@ impl<Hash: Send + 'static> ManualSealApi<Hash> for ManualSeal<Hash> {
&self,
create_empty: bool,
finalize: bool,
parent_hash: Option<Hash>
parent_hash: Option<Hash>,
) -> FutureResult<CreatedBlock<Hash>> {
let mut sink = self.import_block_channel.clone();
let future = async move {
@@ -126,18 +124,22 @@ impl<Hash: Send + 'static> ManualSealApi<Hash> for ManualSeal<Hash> {
};
sink.send(command).await?;
receiver.await?
}.boxed();
}
.boxed();
Box::new(future.map_err(Error::from).compat())
}
fn finalize_block(&self, hash: Hash, justification: Option<EncodedJustification>) -> FutureResult<bool> {
fn finalize_block(
&self,
hash: Hash,
justification: Option<EncodedJustification>,
) -> FutureResult<bool> {
let mut sink = self.import_block_channel.clone();
let future = async move {
let (sender, receiver) = oneshot::channel();
sink.send(
EngineCommand::FinalizeBlock { hash, sender: Some(sender), justification }
).await?;
sink.send(EngineCommand::FinalizeBlock { hash, sender: Some(sender), justification })
.await?;
receiver.await?.map(|_| true)
};
@@ -150,7 +152,7 @@ impl<Hash: Send + 'static> ManualSealApi<Hash> for ManualSeal<Hash> {
/// to the rpc
pub fn send_result<T: std::fmt::Debug>(
sender: &mut Sender<T>,
result: std::result::Result<T, crate::Error>
result: std::result::Result<T, crate::Error>,
) {
if let Some(sender) = sender.take() {
if let Err(err) = sender.send(result) {
@@ -160,7 +162,7 @@ pub fn send_result<T: std::fmt::Debug>(
// instant seal doesn't report errors over rpc, simply log them.
match result {
Ok(r) => log::info!("Instant Seal success: {:?}", r),
Err(e) => log::error!("Instant Seal encountered an error: {}", e)
Err(e) => log::error!("Instant Seal encountered an error: {}", e),
}
}
}
@@ -18,23 +18,21 @@
//! Block sealing utilities
use crate::{Error, rpc, CreatedBlock, ConsensusDataProvider};
use std::sync::Arc;
use sp_runtime::{
traits::{Block as BlockT, Header as HeaderT},
generic::BlockId,
};
use crate::{rpc, ConsensusDataProvider, CreatedBlock, Error};
use futures::prelude::*;
use sp_consensus::{
self, BlockImport, Environment, Proposer, ForkChoiceStrategy,
BlockImportParams, BlockOrigin, ImportResult, SelectChain, StateAction,
};
use sp_blockchain::HeaderBackend;
use std::collections::HashMap;
use std::time::Duration;
use sp_inherents::{CreateInherentDataProviders, InherentDataProvider};
use sp_api::{ProvideRuntimeApi, TransactionFor};
use sc_transaction_pool_api::TransactionPool;
use sp_api::{ProvideRuntimeApi, TransactionFor};
use sp_blockchain::HeaderBackend;
use sp_consensus::{
self, BlockImport, BlockImportParams, BlockOrigin, Environment, ForkChoiceStrategy,
ImportResult, Proposer, SelectChain, StateAction,
};
use sp_inherents::{CreateInherentDataProviders, InherentDataProvider};
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, Header as HeaderT},
};
use std::{collections::HashMap, sync::Arc, time::Duration};
/// max duration for creating a proposal in secs
pub const MAX_PROPOSAL_DURATION: u64 = 10;
@@ -59,7 +57,8 @@ pub struct SealBlockParams<'a, B: BlockT, BI, SC, C: ProvideRuntimeApi<B>, E, TP
/// SelectChain object
pub select_chain: &'a SC,
/// Digest provider for inclusion in blocks.
pub consensus_data_provider: Option<&'a dyn ConsensusDataProvider<B, Transaction = TransactionFor<C, B>>>,
pub consensus_data_provider:
Option<&'a dyn ConsensusDataProvider<B, Transaction = TransactionFor<C, B>>>,
/// block import object
pub block_import: &'a mut BI,
/// Something that can create the inherent data providers.
@@ -97,7 +96,7 @@ pub async fn seal_block<B, BI, SC, C, E, TP, CIDP>(
{
let future = async {
if pool.status().ready == 0 && !create_empty {
return Err(Error::EmptyTransactionPool);
return Err(Error::EmptyTransactionPool)
}
// get the header to build this new block on.
@@ -129,12 +128,15 @@ pub async fn seal_block<B, BI, SC, C, E, TP, CIDP>(
Default::default()
};
let proposal = proposer.propose(
inherent_data.clone(),
digest,
Duration::from_secs(MAX_PROPOSAL_DURATION),
None,
).map_err(|err| Error::StringError(format!("{:?}", err))).await?;
let proposal = proposer
.propose(
inherent_data.clone(),
digest,
Duration::from_secs(MAX_PROPOSAL_DURATION),
None,
)
.map_err(|err| Error::StringError(format!("{:?}", err)))
.await?;
if proposal.block.extrinsics().len() == inherents_len && !create_empty {
return Err(Error::EmptyTransactionPool)
@@ -145,18 +147,17 @@ pub async fn seal_block<B, BI, SC, C, E, TP, CIDP>(
params.body = Some(body);
params.finalized = finalize;
params.fork_choice = Some(ForkChoiceStrategy::LongestChain);
params.state_action = StateAction::ApplyChanges(
sp_consensus::StorageChanges::Changes(proposal.storage_changes)
);
params.state_action = StateAction::ApplyChanges(sp_consensus::StorageChanges::Changes(
proposal.storage_changes,
));
if let Some(digest_provider) = digest_provider {
digest_provider.append_block_import(&parent, &mut params, &inherent_data)?;
}
match block_import.import_block(params, HashMap::new()).await? {
ImportResult::Imported(aux) => {
Ok(CreatedBlock { hash: <B as BlockT>::Header::hash(&header), aux })
},
ImportResult::Imported(aux) =>
Ok(CreatedBlock { hash: <B as BlockT>::Header::hash(&header), aux }),
other => Err(other.into()),
}
};
+88 -107
View File
@@ -41,34 +41,33 @@
mod worker;
pub use crate::worker::{MiningWorker, MiningMetadata, MiningBuild};
pub use crate::worker::{MiningBuild, MiningMetadata, MiningWorker};
use std::{
sync::Arc, borrow::Cow, collections::HashMap, marker::PhantomData,
cmp::Ordering, time::Duration,
};
use codec::{Decode, Encode};
use futures::{Future, StreamExt};
use log::*;
use parking_lot::Mutex;
use sc_client_api::{BlockOf, backend::AuxStore, BlockchainEvents};
use sp_blockchain::{HeaderBackend, ProvideCache, well_known_cache_keys::Id as CacheKeyId};
use sp_block_builder::BlockBuilder as BlockBuilderApi;
use sp_runtime::{Justifications, RuntimeString};
use sp_runtime::generic::{BlockId, Digest, DigestItem};
use sp_runtime::traits::{Block as BlockT, Header as HeaderT};
use prometheus_endpoint::Registry;
use sc_client_api::{self, backend::AuxStore, BlockOf, BlockchainEvents};
use sp_api::ProvideRuntimeApi;
use sp_block_builder::BlockBuilder as BlockBuilderApi;
use sp_blockchain::{well_known_cache_keys::Id as CacheKeyId, HeaderBackend, ProvideCache};
use sp_consensus::{
import_queue::{BasicQueue, BoxBlockImport, BoxJustificationImport, Verifier},
BlockCheckParams, BlockImport, BlockImportParams, BlockOrigin, CanAuthorWith, Environment,
Error as ConsensusError, ForkChoiceStrategy, ImportResult, Proposer, SelectChain, SyncOracle,
};
use sp_consensus_pow::{Seal, TotalDifficulty, POW_ENGINE_ID};
use sp_inherents::{CreateInherentDataProviders, InherentDataProvider};
use sp_consensus::{
BlockImportParams, BlockOrigin, ForkChoiceStrategy, SyncOracle, Environment, Proposer,
SelectChain, Error as ConsensusError, CanAuthorWith, BlockImport, BlockCheckParams, ImportResult,
use sp_runtime::{
generic::{BlockId, Digest, DigestItem},
traits::{Block as BlockT, Header as HeaderT},
Justifications, RuntimeString,
};
use sp_consensus::import_queue::{
BoxBlockImport, BasicQueue, Verifier, BoxJustificationImport,
use std::{
borrow::Cow, cmp::Ordering, collections::HashMap, marker::PhantomData, sync::Arc,
time::Duration,
};
use codec::{Encode, Decode};
use prometheus_endpoint::Registry;
use sc_client_api;
use log::*;
use crate::worker::UntilImportedOrTimeout;
@@ -102,7 +101,7 @@ pub enum Error<B: BlockT> {
CheckInherents(sp_inherents::Error),
#[display(
fmt = "Checking inherents unknown error for identifier: {:?}",
"String::from_utf8_lossy(_0)",
"String::from_utf8_lossy(_0)"
)]
CheckInherentsUnknownError(sp_inherents::InherentIdentifier),
#[display(fmt = "Multiple pre-runtime digests")]
@@ -153,7 +152,8 @@ pub struct PowAux<Difficulty> {
pub total_difficulty: Difficulty,
}
impl<Difficulty> PowAux<Difficulty> where
impl<Difficulty> PowAux<Difficulty>
where
Difficulty: Decode + Default,
{
/// Read the auxiliary from client.
@@ -193,11 +193,7 @@ pub trait PowAlgorithm<B: BlockT> {
/// breaking algorithms will help to protect against selfish mining.
///
/// Returns if the new seal should be considered best block.
fn break_tie(
&self,
_own_seal: &Seal,
_new_seal: &Seal,
) -> bool {
fn break_tie(&self, _own_seal: &Seal, _new_seal: &Seal) -> bool {
false
}
/// Verify that the difficulty is valid against given seal.
@@ -238,7 +234,8 @@ impl<B: BlockT, I: Clone, C, S: Clone, Algorithm: Clone, CAW: Clone, CIDP> Clone
}
}
impl<B, I, C, S, Algorithm, CAW, CIDP> PowBlockImport<B, I, C, S, Algorithm, CAW, CIDP> where
impl<B, I, C, S, Algorithm, CAW, CIDP> PowBlockImport<B, I, C, S, Algorithm, CAW, CIDP>
where
B: BlockT,
I: BlockImport<B, Transaction = sp_api::TransactionFor<C, B>> + Send + Sync,
I::Error: Into<ConsensusError>,
@@ -289,14 +286,15 @@ impl<B, I, C, S, Algorithm, CAW, CIDP> PowBlockImport<B, I, C, S, Algorithm, CAW
return Ok(())
}
let inherent_data = inherent_data_providers.create_inherent_data()
let inherent_data = inherent_data_providers
.create_inherent_data()
.map_err(|e| Error::CreateInherents(e))?;
let inherent_res = self.client.runtime_api().check_inherents(
&block_id,
block,
inherent_data,
).map_err(|e| Error::Client(e.into()))?;
let inherent_res = self
.client
.runtime_api()
.check_inherents(&block_id, block, inherent_data)
.map_err(|e| Error::Client(e.into()))?;
if !inherent_res.ok() {
for (identifier, error) in inherent_res.into_errors() {
@@ -358,20 +356,19 @@ where
self.check_inherents(
check_block.clone(),
BlockId::Hash(parent_hash),
self.create_inherent_data_providers.create_inherent_data_providers(
parent_hash,
(),
).await?,
).await?;
self.create_inherent_data_providers
.create_inherent_data_providers(parent_hash, ())
.await?,
)
.await?;
block.body = Some(check_block.deconstruct().1);
}
let inner_seal = fetch_seal::<B>(block.post_digests.last(), block.header.hash())?;
let intermediate = block.take_intermediate::<PowIntermediate::<Algorithm::Difficulty>>(
INTERMEDIATE_KEY
)?;
let intermediate =
block.take_intermediate::<PowIntermediate<Algorithm::Difficulty>>(INTERMEDIATE_KEY)?;
let difficulty = match intermediate.difficulty {
Some(difficulty) => difficulty,
@@ -401,14 +398,12 @@ where
Ordering::Less => false,
Ordering::Greater => true,
Ordering::Equal => {
let best_inner_seal = fetch_seal::<B>(
best_header.digest().logs.last(),
best_hash,
)?;
let best_inner_seal =
fetch_seal::<B>(best_header.digest().logs.last(), best_hash)?;
self.algorithm.break_tie(&best_inner_seal, &inner_seal)
},
}
},
));
}
@@ -423,35 +418,33 @@ pub struct PowVerifier<B: BlockT, Algorithm> {
}
impl<B: BlockT, Algorithm> PowVerifier<B, Algorithm> {
pub fn new(
algorithm: Algorithm,
) -> Self {
pub fn new(algorithm: Algorithm) -> Self {
Self { algorithm, _marker: PhantomData }
}
fn check_header(
&self,
mut header: B::Header,
) -> Result<(B::Header, DigestItem<B::Hash>), Error<B>> where
) -> Result<(B::Header, DigestItem<B::Hash>), Error<B>>
where
Algorithm: PowAlgorithm<B>,
{
let hash = header.hash();
let (seal, inner_seal) = match header.digest_mut().pop() {
Some(DigestItem::Seal(id, seal)) => {
Some(DigestItem::Seal(id, seal)) =>
if id == POW_ENGINE_ID {
(DigestItem::Seal(id, seal.clone()), seal)
} else {
return Err(Error::WrongEngine(id))
}
},
},
_ => return Err(Error::HeaderUnsealed(hash)),
};
let pre_hash = header.hash();
if !self.algorithm.preliminary_verify(&pre_hash, &inner_seal)?.unwrap_or(true) {
return Err(Error::FailedPreliminaryVerify);
return Err(Error::FailedPreliminaryVerify)
}
Ok((header, seal))
@@ -459,7 +452,8 @@ impl<B: BlockT, Algorithm> PowVerifier<B, Algorithm> {
}
#[async_trait::async_trait]
impl<B: BlockT, Algorithm> Verifier<B> for PowVerifier<B, Algorithm> where
impl<B: BlockT, Algorithm> Verifier<B> for PowVerifier<B, Algorithm>
where
Algorithm: PowAlgorithm<B> + Send + Sync,
Algorithm::Difficulty: 'static + Send,
{
@@ -473,18 +467,15 @@ impl<B: BlockT, Algorithm> Verifier<B> for PowVerifier<B, Algorithm> where
let hash = header.hash();
let (checked_header, seal) = self.check_header(header)?;
let intermediate = PowIntermediate::<Algorithm::Difficulty> {
difficulty: None,
};
let intermediate = PowIntermediate::<Algorithm::Difficulty> { difficulty: None };
let mut import_block = BlockImportParams::new(origin, checked_header);
import_block.post_digests.push(seal);
import_block.body = body;
import_block.justifications = justifications;
import_block.intermediates.insert(
Cow::from(INTERMEDIATE_KEY),
Box::new(intermediate) as Box<_>
);
import_block
.intermediates
.insert(Cow::from(INTERMEDIATE_KEY), Box::new(intermediate) as Box<_>);
import_block.post_hash = Some(hash);
Ok((import_block, None))
@@ -501,10 +492,8 @@ pub fn import_queue<B, Transaction, Algorithm>(
algorithm: Algorithm,
spawner: &impl sp_core::traits::SpawnEssentialNamed,
registry: Option<&Registry>,
) -> Result<
PowImportQueue<B, Transaction>,
sp_consensus::Error
> where
) -> Result<PowImportQueue<B, Transaction>, sp_consensus::Error>
where
B: BlockT,
Transaction: Send + Sync + 'static,
Algorithm: PowAlgorithm<B> + Clone + Send + Sync + 'static,
@@ -512,13 +501,7 @@ pub fn import_queue<B, Transaction, Algorithm>(
{
let verifier = PowVerifier::new(algorithm);
Ok(BasicQueue::new(
verifier,
block_import,
justification_import,
spawner,
registry,
))
Ok(BasicQueue::new(verifier, block_import, justification_import, spawner, registry))
}
/// Start the mining worker for PoW. This function provides the necessary helper functions that can
@@ -573,13 +556,13 @@ where
let task = async move {
loop {
if timer.next().await.is_none() {
break;
break
}
if sync_oracle.is_major_syncing() {
debug!(target: "pow", "Skipping proposal due to sync.");
worker.lock().on_major_syncing();
return;
return
}
let best_header = match select_chain.best_chain().await {
@@ -591,8 +574,8 @@ where
Select best chain error: {:?}",
err
);
return;
}
return
},
};
let best_hash = best_header.hash();
@@ -603,11 +586,11 @@ where
Probably a node update is required!",
err,
);
return;
return
}
if worker.lock().best_hash() == Some(best_hash) {
return;
return
}
// The worker is locked for the duration of the whole proposing period. Within this period,
@@ -622,23 +605,25 @@ where
Fetch difficulty failed: {:?}",
err,
);
return;
return
},
};
let inherent_data_providers =
match create_inherent_data_providers.create_inherent_data_providers(best_hash, ()).await {
Ok(x) => x,
Err(err) => {
warn!(
target: "pow",
"Unable to propose new block for authoring. \
Creating inherent data providers failed: {:?}",
err,
);
return;
},
};
let inherent_data_providers = match create_inherent_data_providers
.create_inherent_data_providers(best_hash, ())
.await
{
Ok(x) => x,
Err(err) => {
warn!(
target: "pow",
"Unable to propose new block for authoring. \
Creating inherent data providers failed: {:?}",
err,
);
return
},
};
let inherent_data = match inherent_data_providers.create_inherent_data() {
Ok(r) => r,
@@ -649,7 +634,7 @@ where
Creating inherent data failed: {:?}",
e,
);
return;
return
},
};
@@ -673,12 +658,10 @@ where
},
};
let proposal = match proposer.propose(
inherent_data,
inherent_digest,
build_time.clone(),
None,
).await {
let proposal = match proposer
.propose(inherent_data, inherent_digest, build_time.clone(), None)
.await
{
Ok(x) => x,
Err(err) => {
warn!(
@@ -714,9 +697,8 @@ fn find_pre_digest<B: BlockT>(header: &B::Header) -> Result<Option<Vec<u8>>, Err
for log in header.digest().logs() {
trace!(target: "pow", "Checking log {:?}, looking for pre runtime digest", log);
match (log, pre_digest.is_some()) {
(DigestItem::PreRuntime(POW_ENGINE_ID, _), true) => {
return Err(Error::MultiplePreRuntimeDigests)
},
(DigestItem::PreRuntime(POW_ENGINE_ID, _), true) =>
return Err(Error::MultiplePreRuntimeDigests),
(DigestItem::PreRuntime(POW_ENGINE_ID, v), false) => {
pre_digest = Some(v.clone());
},
@@ -733,13 +715,12 @@ fn fetch_seal<B: BlockT>(
hash: B::Hash,
) -> Result<Vec<u8>, Error<B>> {
match digest {
Some(DigestItem::Seal(id, seal)) => {
Some(DigestItem::Seal(id, seal)) =>
if id == &POW_ENGINE_ID {
Ok(seal.clone())
} else {
return Err(Error::<B>::WrongEngine(*id).into())
}
},
},
_ => return Err(Error::<B>::HeaderUnsealed(hash).into()),
}
}
+26 -29
View File
@@ -16,20 +16,25 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::{pin::Pin, time::Duration, collections::HashMap, borrow::Cow};
use futures::{
prelude::*,
task::{Context, Poll},
};
use futures_timer::Delay;
use log::*;
use sc_client_api::ImportNotifications;
use sp_consensus::{Proposal, BlockOrigin, BlockImportParams, StorageChanges,
StateAction, import_queue::BoxBlockImport};
use sp_consensus::{
import_queue::BoxBlockImport, BlockImportParams, BlockOrigin, Proposal, StateAction,
StorageChanges,
};
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, Header as HeaderT},
DigestItem,
};
use futures::{prelude::*, task::{Context, Poll}};
use futures_timer::Delay;
use log::*;
use std::{borrow::Cow, collections::HashMap, pin::Pin, time::Duration};
use crate::{INTERMEDIATE_KEY, POW_ENGINE_ID, Seal, PowAlgorithm, PowIntermediate};
use crate::{PowAlgorithm, PowIntermediate, Seal, INTERMEDIATE_KEY, POW_ENGINE_ID};
/// Mining metadata. This is the information needed to start an actual mining loop.
#[derive(Clone, Eq, PartialEq)]
@@ -49,7 +54,7 @@ pub struct MiningBuild<
Block: BlockT,
Algorithm: PowAlgorithm<Block>,
C: sp_api::ProvideRuntimeApi<Block>,
Proof
Proof,
> {
/// Mining metadata.
pub metadata: MiningMetadata<Block::Hash, Algorithm::Difficulty>,
@@ -90,10 +95,7 @@ where
self.build = None;
}
pub(crate) fn on_build(
&mut self,
build: MiningBuild<Block, Algorithm, C, Proof>,
) {
pub(crate) fn on_build(&mut self, build: MiningBuild<Block, Algorithm, C, Proof>) {
self.build = Some(build);
}
@@ -137,23 +139,25 @@ where
let mut import_block = BlockImportParams::new(BlockOrigin::Own, header);
import_block.post_digests.push(seal);
import_block.body = Some(body);
import_block.state_action = StateAction::ApplyChanges(
StorageChanges::Changes(build.proposal.storage_changes)
);
import_block.state_action =
StateAction::ApplyChanges(StorageChanges::Changes(build.proposal.storage_changes));
let intermediate = PowIntermediate::<Algorithm::Difficulty> {
difficulty: Some(build.metadata.difficulty),
};
import_block.intermediates.insert(
Cow::from(INTERMEDIATE_KEY),
Box::new(intermediate) as Box<_>,
);
import_block
.intermediates
.insert(Cow::from(INTERMEDIATE_KEY), Box::new(intermediate) as Box<_>);
let header = import_block.post_header();
match self.block_import.import_block(import_block, HashMap::default()).await {
Ok(res) => {
res.handle_justification(&header.hash(), *header.number(), &mut self.justification_sync_link);
res.handle_justification(
&header.hash(),
*header.number(),
&mut self.justification_sync_link,
);
info!(
target: "pow",
@@ -190,15 +194,8 @@ pub struct UntilImportedOrTimeout<Block: BlockT> {
impl<Block: BlockT> UntilImportedOrTimeout<Block> {
/// Create a new stream using the given import notification and timeout duration.
pub fn new(
import_notifications: ImportNotifications<Block>,
timeout: Duration,
) -> Self {
Self {
import_notifications,
timeout,
inner_delay: None,
}
pub fn new(import_notifications: ImportNotifications<Block>, timeout: Duration) -> Self {
Self { import_notifications, timeout, inner_delay: None }
}
}
@@ -18,9 +18,9 @@
//! Schema for slots in the aux-db.
use codec::{Encode, Decode};
use codec::{Decode, Encode};
use sc_client_api::backend::AuxStore;
use sp_blockchain::{Result as ClientResult, Error as ClientError};
use sp_blockchain::{Error as ClientError, Result as ClientResult};
use sp_consensus_slots::{EquivocationProof, Slot};
use sp_runtime::traits::Header;
@@ -33,17 +33,17 @@ pub const MAX_SLOT_CAPACITY: u64 = 1000;
pub const PRUNING_BOUND: u64 = 2 * MAX_SLOT_CAPACITY;
fn load_decode<C, T>(backend: &C, key: &[u8]) -> ClientResult<Option<T>>
where
C: AuxStore,
T: Decode,
where
C: AuxStore,
T: Decode,
{
match backend.get_aux(key)? {
None => Ok(None),
Some(t) => T::decode(&mut &t[..])
.map_err(
|e| ClientError::Backend(format!("Slots DB is corrupted. Decode error: {}", e)),
)
.map(Some)
.map_err(|e| {
ClientError::Backend(format!("Slots DB is corrupted. Decode error: {}", e))
})
.map(Some),
}
}
@@ -57,14 +57,14 @@ pub fn check_equivocation<C, H, P>(
header: &H,
signer: &P,
) -> ClientResult<Option<EquivocationProof<H, P>>>
where
H: Header,
C: AuxStore,
P: Clone + Encode + Decode + PartialEq,
where
H: Header,
C: AuxStore,
P: Clone + Encode + Decode + PartialEq,
{
// We don't check equivocations for old headers out of our capacity.
if slot_now.saturating_sub(*slot) > Slot::from(MAX_SLOT_CAPACITY) {
return Ok(None);
return Ok(None)
}
// Key for this slot.
@@ -72,17 +72,16 @@ pub fn check_equivocation<C, H, P>(
slot.using_encoded(|s| curr_slot_key.extend(s));
// Get headers of this slot.
let mut headers_with_sig = load_decode::<_, Vec<(H, P)>>(backend, &curr_slot_key[..])?
.unwrap_or_else(Vec::new);
let mut headers_with_sig =
load_decode::<_, Vec<(H, P)>>(backend, &curr_slot_key[..])?.unwrap_or_else(Vec::new);
// Get first slot saved.
let slot_header_start = SLOT_HEADER_START.to_vec();
let first_saved_slot = load_decode::<_, Slot>(backend, &slot_header_start[..])?
.unwrap_or(slot);
let first_saved_slot = load_decode::<_, Slot>(backend, &slot_header_start[..])?.unwrap_or(slot);
if slot_now < first_saved_slot {
// The code below assumes that slots will be visited sequentially.
return Ok(None);
return Ok(None)
}
for (prev_header, prev_signer) in headers_with_sig.iter() {
@@ -96,7 +95,7 @@ pub fn check_equivocation<C, H, P>(
offender: signer.clone(),
first_header: prev_header.clone(),
second_header: header.clone(),
}));
}))
} else {
// We don't need to continue in case of duplicated header,
// since it's already saved and a possible equivocation
@@ -135,12 +134,11 @@ pub fn check_equivocation<C, H, P>(
#[cfg(test)]
mod test {
use sp_core::{sr25519, Pair};
use sp_core::hash::H256;
use sp_runtime::testing::{Header as HeaderTest, Digest as DigestTest};
use sp_core::{hash::H256, sr25519, Pair};
use sp_runtime::testing::{Digest as DigestTest, Header as HeaderTest};
use substrate_test_runtime_client;
use super::{MAX_SLOT_CAPACITY, PRUNING_BOUND, check_equivocation};
use super::{check_equivocation, MAX_SLOT_CAPACITY, PRUNING_BOUND};
fn create_header(number: u64) -> HeaderTest {
// so that different headers for the same number get different hashes
@@ -151,7 +149,7 @@ mod test {
number,
state_root: Default::default(),
extrinsics_root: Default::default(),
digest: DigestTest { logs: vec![], },
digest: DigestTest { logs: vec![] },
};
header
@@ -171,79 +169,55 @@ mod test {
let header6 = create_header(3); // @ slot 4
// It's ok to sign same headers.
assert!(
check_equivocation(
&client,
2.into(),
2.into(),
&header1,
&public,
).unwrap().is_none(),
);
assert!(check_equivocation(&client, 2.into(), 2.into(), &header1, &public,)
.unwrap()
.is_none(),);
assert!(
check_equivocation(
&client,
3.into(),
2.into(),
&header1,
&public,
).unwrap().is_none(),
);
assert!(check_equivocation(&client, 3.into(), 2.into(), &header1, &public,)
.unwrap()
.is_none(),);
// But not two different headers at the same slot.
assert!(
check_equivocation(
&client,
4.into(),
2.into(),
&header2,
&public,
).unwrap().is_some(),
);
assert!(check_equivocation(&client, 4.into(), 2.into(), &header2, &public,)
.unwrap()
.is_some(),);
// Different slot is ok.
assert!(
check_equivocation(
&client,
5.into(),
4.into(),
&header3,
&public,
).unwrap().is_none(),
);
assert!(check_equivocation(&client, 5.into(), 4.into(), &header3, &public,)
.unwrap()
.is_none(),);
// Here we trigger pruning and save header 4.
assert!(
check_equivocation(
&client,
(PRUNING_BOUND + 2).into(),
(MAX_SLOT_CAPACITY + 4).into(),
&header4,
&public,
).unwrap().is_none(),
);
assert!(check_equivocation(
&client,
(PRUNING_BOUND + 2).into(),
(MAX_SLOT_CAPACITY + 4).into(),
&header4,
&public,
)
.unwrap()
.is_none(),);
// This fails because header 5 is an equivocation of header 4.
assert!(
check_equivocation(
&client,
(PRUNING_BOUND + 3).into(),
(MAX_SLOT_CAPACITY + 4).into(),
&header5,
&public,
).unwrap().is_some(),
);
assert!(check_equivocation(
&client,
(PRUNING_BOUND + 3).into(),
(MAX_SLOT_CAPACITY + 4).into(),
&header5,
&public,
)
.unwrap()
.is_some(),);
// This is ok because we pruned the corresponding header. Shows that we are pruning.
assert!(
check_equivocation(
&client,
(PRUNING_BOUND + 4).into(),
4.into(),
&header6,
&public,
).unwrap().is_none(),
);
assert!(check_equivocation(
&client,
(PRUNING_BOUND + 4).into(),
4.into(),
&header6,
&public,
)
.unwrap()
.is_none(),);
}
}
+144 -175
View File
@@ -25,19 +25,19 @@
#![forbid(unsafe_code)]
#![warn(missing_docs)]
mod slots;
mod aux_schema;
mod slots;
pub use aux_schema::{check_equivocation, MAX_SLOT_CAPACITY, PRUNING_BOUND};
pub use slots::SlotInfo;
use slots::Slots;
pub use aux_schema::{check_equivocation, MAX_SLOT_CAPACITY, PRUNING_BOUND};
use std::{fmt::Debug, ops::Deref, time::Duration};
use codec::{Decode, Encode};
use futures::{future::Either, Future, TryFutureExt};
use futures_timer::Delay;
use log::{debug, error, info, warn};
use sp_api::{ProvideRuntimeApi, ApiRef};
use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO, CONSENSUS_WARN};
use sp_api::{ApiRef, ProvideRuntimeApi};
use sp_arithmetic::traits::BaseArithmetic;
use sp_consensus::{
BlockImport, CanAuthorWith, JustificationSyncLink, Proposer, SelectChain, SlotData, SyncOracle,
@@ -46,10 +46,10 @@ use sp_consensus_slots::Slot;
use sp_inherents::CreateInherentDataProviders;
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, Header as HeaderT, HashFor, NumberFor}
traits::{Block as BlockT, HashFor, Header as HeaderT, NumberFor},
};
use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_WARN, CONSENSUS_INFO};
use sp_timestamp::Timestamp;
use std::{fmt::Debug, ops::Deref, time::Duration};
/// The changes that need to applied to the storage to create the state for a block.
///
@@ -76,10 +76,7 @@ pub trait SlotWorker<B: BlockT, Proof> {
///
/// Returns a future that resolves to a [`SlotResult`] iff a block was successfully built in
/// the slot. Otherwise `None` is returned.
async fn on_slot(
&mut self,
slot_info: SlotInfo<B>,
) -> Option<SlotResult<B, Proof>>;
async fn on_slot(&mut self, slot_info: SlotInfo<B>) -> Option<SlotResult<B, Proof>>;
}
/// A skeleton implementation for `SlotWorker` which tries to claim a slot at
@@ -89,7 +86,8 @@ pub trait SlotWorker<B: BlockT, Proof> {
pub trait SimpleSlotWorker<B: BlockT> {
/// A handle to a `BlockImport`.
type BlockImport: BlockImport<B, Transaction = <Self::Proposer as Proposer<B>>::Transaction>
+ Send + 'static;
+ Send
+ 'static;
/// A handle to a `SyncOracle`.
type SyncOracle: SyncOracle;
@@ -100,7 +98,9 @@ pub trait SimpleSlotWorker<B: BlockT> {
/// The type of future resolving to the proposer.
type CreateProposer: Future<Output = Result<Self::Proposer, sp_consensus::Error>>
+ Send + Unpin + 'static;
+ Send
+ Unpin
+ 'static;
/// The type of proposer to use to build blocks.
type Proposer: Proposer<B> + Send;
@@ -139,12 +139,7 @@ pub trait SimpleSlotWorker<B: BlockT> {
/// Notifies the given slot. Similar to `claim_slot`, but will be called no matter whether we
/// need to author blocks or not.
fn notify_slot(
&self,
_header: &B::Header,
_slot: Slot,
_epoch_data: &Self::EpochData,
) {}
fn notify_slot(&self, _header: &B::Header, _slot: Slot, _epoch_data: &Self::EpochData) {}
/// Return the pre digest data to include in a block authored with the given claim.
fn pre_digest_data(
@@ -154,18 +149,24 @@ pub trait SimpleSlotWorker<B: BlockT> {
) -> Vec<sp_runtime::DigestItem<B::Hash>>;
/// Returns a function which produces a `BlockImportParams`.
fn block_import_params(&self) -> Box<
fn block_import_params(
&self,
) -> Box<
dyn Fn(
B::Header,
&B::Hash,
Vec<B::Extrinsic>,
StorageChanges<<Self::BlockImport as BlockImport<B>>::Transaction, B>,
Self::Claim,
Self::EpochData,
) -> Result<
sp_consensus::BlockImportParams<B, <Self::BlockImport as BlockImport<B>>::Transaction>,
sp_consensus::Error
> + Send + 'static
B::Header,
&B::Hash,
Vec<B::Extrinsic>,
StorageChanges<<Self::BlockImport as BlockImport<B>>::Transaction, B>,
Self::Claim,
Self::EpochData,
) -> Result<
sp_consensus::BlockImportParams<
B,
<Self::BlockImport as BlockImport<B>>::Transaction,
>,
sp_consensus::Error,
> + Send
+ 'static,
>;
/// Whether to force authoring if offline.
@@ -194,10 +195,7 @@ pub trait SimpleSlotWorker<B: BlockT> {
fn telemetry(&self) -> Option<TelemetryHandle>;
/// Remaining duration for proposing.
fn proposing_remaining_duration(
&self,
slot_info: &SlotInfo<B>,
) -> Duration;
fn proposing_remaining_duration(&self, slot_info: &SlotInfo<B>) -> Duration;
/// Implements [`SlotWorker::on_slot`].
async fn on_slot(
@@ -213,8 +211,7 @@ pub trait SimpleSlotWorker<B: BlockT> {
let proposing_remaining = if proposing_remaining_duration == Duration::default() {
debug!(
target: logging_target,
"Skipping proposal slot {} since there's no time left to propose",
slot,
"Skipping proposal slot {} since there's no time left to propose", slot,
);
return None
@@ -240,8 +237,8 @@ pub trait SimpleSlotWorker<B: BlockT> {
"err" => ?err,
);
return None;
}
return None
},
};
self.notify_slot(&slot_info.chain_head, slot, &epoch_data);
@@ -260,13 +257,13 @@ pub trait SimpleSlotWorker<B: BlockT> {
"authorities_len" => authorities_len,
);
return None;
return None
}
let claim = self.claim_slot(&slot_info.chain_head, slot, &epoch_data)?;
if self.should_backoff(slot, &slot_info.chain_head) {
return None;
return None
}
debug!(
@@ -289,9 +286,7 @@ pub trait SimpleSlotWorker<B: BlockT> {
Err(err) => {
warn!(
target: logging_target,
"Unable to author block in slot {:?}: {:?}",
slot,
err,
"Unable to author block in slot {:?}: {:?}", slot, err,
);
telemetry!(
@@ -303,7 +298,7 @@ pub trait SimpleSlotWorker<B: BlockT> {
);
return None
}
},
};
let logs = self.pre_digest_data(slot, &claim);
@@ -311,34 +306,29 @@ pub trait SimpleSlotWorker<B: BlockT> {
// deadline our production to 98% of the total time left for proposing. As we deadline
// the proposing below to the same total time left, the 2% margin should be enough for
// the result to be returned.
let proposing = proposer.propose(
slot_info.inherent_data,
sp_runtime::generic::Digest {
logs,
},
proposing_remaining_duration.mul_f32(0.98),
None,
).map_err(|e| sp_consensus::Error::ClientImport(format!("{:?}", e)));
let proposing = proposer
.propose(
slot_info.inherent_data,
sp_runtime::generic::Digest { logs },
proposing_remaining_duration.mul_f32(0.98),
None,
)
.map_err(|e| sp_consensus::Error::ClientImport(format!("{:?}", e)));
let proposal = match futures::future::select(proposing, proposing_remaining).await {
Either::Left((Ok(p), _)) => p,
Either::Left((Err(err), _)) => {
warn!(
target: logging_target,
"Proposing failed: {:?}",
err,
);
warn!(target: logging_target, "Proposing failed: {:?}", err,);
return None
},
Either::Right(_) => {
info!(
target: logging_target,
"⌛️ Discarding proposal for slot {}; block production took too long",
slot,
"⌛️ Discarding proposal for slot {}; block production took too long", slot,
);
// If the node was compiled with debug, tell the user to use release optimizations.
#[cfg(build_type="debug")]
#[cfg(build_type = "debug")]
info!(
target: logging_target,
"👉 Recompile your node in `--release` mode to mitigate this problem.",
@@ -373,14 +363,10 @@ pub trait SimpleSlotWorker<B: BlockT> {
) {
Ok(bi) => bi,
Err(err) => {
warn!(
target: logging_target,
"Failed to create block import params: {:?}",
err,
);
warn!(target: logging_target, "Failed to create block import params: {:?}", err,);
return None
}
},
};
info!(
@@ -401,17 +387,14 @@ pub trait SimpleSlotWorker<B: BlockT> {
);
let header = block_import_params.post_header();
match block_import
.import_block(block_import_params, Default::default())
.await
{
match block_import.import_block(block_import_params, Default::default()).await {
Ok(res) => {
res.handle_justification(
&header.hash(),
*header.number(),
self.justification_sync_link(),
);
}
},
Err(err) => {
warn!(
target: logging_target,
@@ -425,18 +408,17 @@ pub trait SimpleSlotWorker<B: BlockT> {
"hash" => ?parent_hash,
"err" => ?err,
);
}
},
}
Some(SlotResult {
block: B::new(header, body),
storage_proof,
})
Some(SlotResult { block: B::new(header, body), storage_proof })
}
}
#[async_trait::async_trait]
impl<B: BlockT, T: SimpleSlotWorker<B> + Send> SlotWorker<B, <T::Proposer as Proposer<B>>::Proof> for T {
impl<B: BlockT, T: SimpleSlotWorker<B> + Send> SlotWorker<B, <T::Proposer as Proposer<B>>::Proof>
for T
{
async fn on_slot(
&mut self,
slot_info: SlotInfo<B>,
@@ -496,8 +478,7 @@ pub async fn start_slot_worker<B, C, W, T, SO, CIDP, CAW, Proof>(
mut sync_oracle: SO,
create_inherent_data_providers: CIDP,
can_author_with: CAW,
)
where
) where
B: BlockT,
C: SelectChain<B>,
W: SlotWorker<B, Proof>,
@@ -509,28 +490,25 @@ where
{
let SlotDuration(slot_duration) = slot_duration;
let mut slots = Slots::new(
slot_duration.slot_duration(),
create_inherent_data_providers,
client,
);
let mut slots =
Slots::new(slot_duration.slot_duration(), create_inherent_data_providers, client);
loop {
let slot_info = match slots.next_slot().await {
Ok(r) => r,
Err(e) => {
warn!(target: "slots", "Error while polling for next slot: {:?}", e);
return;
}
return
},
};
if sync_oracle.is_major_syncing() {
debug!(target: "slots", "Skipping proposal slot due to sync.");
continue;
continue
}
if let Err(err) = can_author_with
.can_author_with(&BlockId::Hash(slot_info.chain_head.hash()))
if let Err(err) =
can_author_with.can_author_with(&BlockId::Hash(slot_info.chain_head.hash()))
{
warn!(
target: "slots",
@@ -559,7 +537,10 @@ pub enum CheckedHeader<H, S> {
#[derive(Debug, thiserror::Error)]
#[allow(missing_docs)]
pub enum Error<T> where T: Debug {
pub enum Error<T>
where
T: Debug,
{
#[error("Slot duration is invalid: {0:?}")]
SlotDurationInvalid(SlotDuration<T>),
}
@@ -591,25 +572,23 @@ impl<T: Clone + Send + Sync + 'static> SlotDuration<T> {
///
/// `slot_key` is marked as `'static`, as it should really be a
/// compile-time constant.
pub fn get_or_compute<B: BlockT, C, CB>(client: &C, cb: CB) -> sp_blockchain::Result<Self> where
pub fn get_or_compute<B: BlockT, C, CB>(client: &C, cb: CB) -> sp_blockchain::Result<Self>
where
C: sc_client_api::backend::AuxStore + sc_client_api::UsageProvider<B>,
C: ProvideRuntimeApi<B>,
CB: FnOnce(ApiRef<C::Api>, &BlockId<B>) -> sp_blockchain::Result<T>,
T: SlotData + Encode + Decode + Debug,
{
let slot_duration = match client.get_aux(T::SLOT_KEY)? {
Some(v) => <T as codec::Decode>::decode(&mut &v[..])
.map(SlotDuration)
.map_err(|_| {
sp_blockchain::Error::Backend({
error!(target: "slots", "slot duration kept in invalid format");
"slot duration kept in invalid format".to_string()
})
}),
Some(v) => <T as codec::Decode>::decode(&mut &v[..]).map(SlotDuration).map_err(|_| {
sp_blockchain::Error::Backend({
error!(target: "slots", "slot duration kept in invalid format");
"slot duration kept in invalid format".to_string()
})
}),
None => {
let best_hash = client.usage_info().chain.best_hash;
let slot_duration =
cb(client.runtime_api(), &BlockId::hash(best_hash))?;
let slot_duration = cb(client.runtime_api(), &BlockId::hash(best_hash))?;
info!(
"⏱ Loaded block-time = {:?} from block {:?}",
@@ -621,11 +600,13 @@ impl<T: Clone + Send + Sync + 'static> SlotDuration<T> {
.using_encoded(|s| client.insert_aux(&[(T::SLOT_KEY, &s[..])], &[]))?;
Ok(SlotDuration(slot_duration))
}
},
}?;
if slot_duration.slot_duration() == Default::default() {
return Err(sp_blockchain::Error::Application(Box::new(Error::SlotDurationInvalid(slot_duration))))
return Err(sp_blockchain::Error::Application(Box::new(Error::SlotDurationInvalid(
slot_duration,
))))
}
Ok(slot_duration)
@@ -687,9 +668,7 @@ pub fn proposing_remaining_duration<Block: BlockT>(
) -> Duration {
use sp_runtime::traits::Zero;
let proposing_duration = slot_info
.duration
.mul_f32(block_proposal_slot_portion.get());
let proposing_duration = slot_info.duration.mul_f32(block_proposal_slot_portion.get());
let slot_remaining = slot_info
.ends_at
@@ -700,7 +679,7 @@ pub fn proposing_remaining_duration<Block: BlockT>(
// If parent is genesis block, we don't require any lenience factor.
if slot_info.chain_head.number().is_zero() {
return proposing_duration;
return proposing_duration
}
let parent_slot = match parent_slot {
@@ -723,9 +702,7 @@ pub fn proposing_remaining_duration<Block: BlockT>(
if let Some(ref max_block_proposal_slot_portion) = max_block_proposal_slot_portion {
std::cmp::min(
lenient_proposing_duration,
slot_info
.duration
.mul_f32(max_block_proposal_slot_portion.get()),
slot_info.duration.mul_f32(max_block_proposal_slot_portion.get()),
)
} else {
lenient_proposing_duration
@@ -853,7 +830,7 @@ impl<N: BaseArithmetic> Default for BackoffAuthoringOnFinalizedHeadLagging<N> {
impl<N> BackoffAuthoringBlocksStrategy<N> for BackoffAuthoringOnFinalizedHeadLagging<N>
where
N: BaseArithmetic + Copy
N: BaseArithmetic + Copy,
{
fn should_backoff(
&self,
@@ -865,12 +842,12 @@ where
) -> bool {
// This should not happen, but we want to keep the previous behaviour if it does.
if slot_now <= chain_head_slot {
return false;
return false
}
let unfinalized_block_length = chain_head_number - finalized_number;
let interval = unfinalized_block_length.saturating_sub(self.unfinalized_slack)
/ self.authoring_bias;
let interval =
unfinalized_block_length.saturating_sub(self.unfinalized_slack) / self.authoring_bias;
let interval = interval.min(self.max_interval);
// We're doing arithmetic between block and slot numbers.
@@ -906,9 +883,9 @@ impl<N> BackoffAuthoringBlocksStrategy<N> for () {
#[cfg(test)]
mod test {
use super::*;
use sp_api::NumberFor;
use std::time::{Duration, Instant};
use substrate_test_runtime_client::runtime::{Block, Header};
use sp_api::NumberFor;
const SLOT_DURATION: Duration = Duration::from_millis(6000);
@@ -945,10 +922,7 @@ mod test {
}
// but we cap it to a maximum of 20 slots
assert_eq!(
super::slot_lenience_linear(1u64.into(), &slot(23)),
Some(SLOT_DURATION * 20),
);
assert_eq!(super::slot_lenience_linear(1u64.into(), &slot(23)), Some(SLOT_DURATION * 20),);
}
#[test]
@@ -1041,7 +1015,15 @@ mod test {
let slot_now = 2;
let should_backoff: Vec<bool> = (slot_now..1000)
.map(|s| strategy.should_backoff(head_number, head_slot.into(), finalized_number, s.into(), "slots"))
.map(|s| {
strategy.should_backoff(
head_number,
head_slot.into(),
finalized_number,
s.into(),
"slots",
)
})
.collect();
// Should always be false, since the head isn't advancing
@@ -1105,7 +1087,15 @@ mod test {
let max_interval = strategy.max_interval;
let should_backoff: Vec<bool> = (slot_now..200)
.map(|s| strategy.should_backoff(head_number, head_slot.into(), finalized_number, s.into(), "slots"))
.map(|s| {
strategy.should_backoff(
head_number,
head_slot.into(),
finalized_number,
s.into(),
"slots",
)
})
.collect();
// Should backoff (true) until we are `max_interval` number of slots ahead of the chain
@@ -1123,11 +1113,7 @@ mod test {
};
let finalized_number = 2;
let mut head_state = HeadState {
head_number: 4,
head_slot: 10,
slot_now: 11,
};
let mut head_state = HeadState { head_number: 4, head_slot: 10, slot_now: 11 };
let should_backoff = |head_state: &HeadState| -> bool {
<dyn BackoffAuthoringBlocksStrategy<NumberFor<Block>>>::should_backoff(
@@ -1155,32 +1141,27 @@ mod test {
// Gradually start to backoff more and more frequently
let expected = [
false, false, false, false, false, // no effect
true, false,
true, false, // 1:1
true, true, false,
true, true, false, // 2:1
true, true, true, false,
true, true, true, false, // 3:1
true, true, true, true, false,
true, true, true, true, false, // 4:1
true, true, true, true, true, false,
true, true, true, true, true, false, // 5:1
true, true, true, true, true, true, false,
true, true, true, true, true, true, false, // 6:1
true, true, true, true, true, true, true, false,
true, true, true, true, true, true, true, false, // 7:1
true, true, true, true, true, true, true, true, false,
true, true, true, true, true, true, true, true, false, // 8:1
true, true, true, true, true, true, true, true, true, false,
true, true, true, true, true, true, true, true, true, false, // 9:1
true, true, true, true, true, true, true, true, true, true, false,
true, true, true, true, true, true, true, true, true, true, false, // 10:1
true, true, true, true, true, true, true, true, true, true, true, false,
true, true, true, true, true, true, true, true, true, true, true, false, // 11:1
true, true, true, true, true, true, true, true, true, true, true, true, false,
true, true, true, true, true, true, true, true, true, true, true, true, false, // 12:1
true, false, true, false, // 1:1
true, true, false, true, true, false, // 2:1
true, true, true, false, true, true, true, false, // 3:1
true, true, true, true, false, true, true, true, true, false, // 4:1
true, true, true, true, true, false, true, true, true, true, true, false, // 5:1
true, true, true, true, true, true, false, true, true, true, true, true, true,
false, // 6:1
true, true, true, true, true, true, true, false, true, true, true, true, true, true,
true, false, // 7:1
true, true, true, true, true, true, true, true, false, true, true, true, true, true,
true, true, true, false, // 8:1
true, true, true, true, true, true, true, true, true, false, true, true, true, true,
true, true, true, true, true, false, // 9:1
true, true, true, true, true, true, true, true, true, true, false, true, true, true,
true, true, true, true, true, true, true, false, // 10:1
true, true, true, true, true, true, true, true, true, true, true, false, true, true,
true, true, true, true, true, true, true, true, true, false, // 11:1
true, true, true, true, true, true, true, true, true, true, true, true, false, true,
true, true, true, true, true, true, true, true, true, true, true, false, // 12:1
true, true, true, true,
];
];
assert_eq!(backoff.as_slice(), &expected[..]);
}
@@ -1195,11 +1176,7 @@ mod test {
let finalized_number = 2;
let starting_slot = 11;
let mut head_state = HeadState {
head_number: 4,
head_slot: 10,
slot_now: starting_slot,
};
let mut head_state = HeadState { head_number: 4, head_slot: 10, slot_now: starting_slot };
let should_backoff = |head_state: &HeadState| -> bool {
<dyn BackoffAuthoringBlocksStrategy<NumberFor<Block>>>::should_backoff(
@@ -1240,30 +1217,22 @@ mod test {
assert_eq!(last_slot - last_two_claimed.next().unwrap(), 92);
assert_eq!(last_slot - last_two_claimed.next().unwrap(), 92 + expected_distance);
let intervals: Vec<_> = slots_claimed
.windows(2)
.map(|x| x[1] - x[0])
.collect();
let intervals: Vec<_> = slots_claimed.windows(2).map(|x| x[1] - x[0]).collect();
// The key thing is that the distance between claimed slots is capped to `max_interval + 1`
// assert_eq!(max_observed_interval, Some(&expected_distance));
assert_eq!(intervals.iter().max(), Some(&expected_distance));
// But lets assert all distances, which we expect to grow linearly until `max_interval + 1`
let expected_intervals: Vec<_> = (0..497)
.map(|i| (i/2).max(1).min(expected_distance) )
.collect();
let expected_intervals: Vec<_> =
(0..497).map(|i| (i / 2).max(1).min(expected_distance)).collect();
assert_eq!(intervals, expected_intervals);
}
fn run_until_max_interval(param: BackoffAuthoringOnFinalizedHeadLagging<u64>) -> (u64, u64) {
let finalized_number = 0;
let mut head_state = HeadState {
head_number: 0,
head_slot: 0,
slot_now: 1,
};
let mut head_state = HeadState { head_number: 0, head_slot: 0, slot_now: 1 };
let should_backoff = |head_state: &HeadState| -> bool {
<dyn BackoffAuthoringBlocksStrategy<NumberFor<Block>>>::should_backoff(
@@ -1277,8 +1246,8 @@ mod test {
};
// Number of blocks until we reach the max interval
let block_for_max_interval
= param.max_interval * param.authoring_bias + param.unfinalized_slack;
let block_for_max_interval =
param.max_interval * param.authoring_bias + param.unfinalized_slack;
while head_state.head_number < block_for_max_interval {
if should_backoff(&head_state) {
@@ -1294,15 +1263,15 @@ mod test {
}
// Denoting
// C: unfinalized_slack
// M: authoring_bias
// X: max_interval
// C: unfinalized_slack
// M: authoring_bias
// X: max_interval
// then the number of slots to reach the max interval can be computed from
// (start_slot + C) + M * sum(n, 1, X)
// (start_slot + C) + M * sum(n, 1, X)
// or
// (start_slot + C) + M * X*(X+1)/2
// (start_slot + C) + M * X*(X+1)/2
fn expected_time_to_reach_max_interval(
param: &BackoffAuthoringOnFinalizedHeadLagging<u64>
param: &BackoffAuthoringOnFinalizedHeadLagging<u64>,
) -> (u64, u64) {
let c = param.unfinalized_slack;
let m = param.authoring_bias;
+12 -17
View File
@@ -20,23 +20,21 @@
//!
//! This is used instead of `futures_timer::Interval` because it was unreliable.
use super::{Slot, InherentDataProviderExt};
use super::{InherentDataProviderExt, Slot};
use sp_consensus::{Error, SelectChain};
use sp_inherents::{InherentData, CreateInherentDataProviders, InherentDataProvider};
use sp_inherents::{CreateInherentDataProviders, InherentData, InherentDataProvider};
use sp_runtime::traits::{Block as BlockT, Header as HeaderT};
use std::time::{Duration, Instant};
use futures_timer::Delay;
use std::time::{Duration, Instant};
/// Returns current duration since unix epoch.
pub fn duration_now() -> Duration {
use std::time::SystemTime;
let now = SystemTime::now();
now.duration_since(SystemTime::UNIX_EPOCH).unwrap_or_else(|e| panic!(
"Current time {:?} is before unix epoch. Something is wrong: {:?}",
now,
e,
))
now.duration_since(SystemTime::UNIX_EPOCH).unwrap_or_else(|e| {
panic!("Current time {:?} is before unix epoch. Something is wrong: {:?}", now, e,)
})
}
/// Returns the duration until the next slot from now.
@@ -104,11 +102,7 @@ pub(crate) struct Slots<Block, C, IDP> {
impl<Block, C, IDP> Slots<Block, C, IDP> {
/// Create a new `Slots` stream.
pub fn new(
slot_duration: Duration,
create_inherent_data_providers: IDP,
client: C,
) -> Self {
pub fn new(slot_duration: Duration, create_inherent_data_providers: IDP, client: C) -> Self {
Slots {
last_slot: 0.into(),
slot_duration,
@@ -135,7 +129,7 @@ where
// schedule wait.
let wait_dur = time_until_next_slot(self.slot_duration);
Some(Delay::new(wait_dur))
}
},
Some(d) => Some(d),
};
@@ -161,11 +155,12 @@ where
);
// Let's try at the next slot..
self.inner_delay.take();
continue;
}
continue
},
};
let inherent_data_providers = self.create_inherent_data_providers
let inherent_data_providers = self
.create_inherent_data_providers
.create_inherent_data_providers(chain_head.hash(), ())
.await?;
+3 -2
View File
@@ -19,7 +19,7 @@
//! Uncles functionality for Substrate.
use sc_client_api::ProvideUncles;
use sp_runtime::{traits::Block as BlockT, generic::BlockId};
use sp_runtime::{generic::BlockId, traits::Block as BlockT};
#[derive(Debug, thiserror::Error)]
pub enum Error<B: BlockT> {
@@ -34,7 +34,8 @@ const MAX_UNCLE_GENERATIONS: u32 = 8;
pub fn create_uncles_inherent_data_provider<B, C>(
client: &C,
parent: B::Hash,
) -> Result<sp_authorship::InherentDataProvider<B::Header>, sc_client_api::blockchain::Error> where
) -> Result<sp_authorship::InherentDataProvider<B::Header>, sc_client_api::blockchain::Error>
where
B: BlockT,
C: ProvideUncles<B>,
{
+121 -80
View File
@@ -18,27 +18,31 @@
//! State backend that's useful for benchmarking
use std::sync::Arc;
use std::cell::{Cell, RefCell};
use std::collections::HashMap;
use std::{
cell::{Cell, RefCell},
collections::HashMap,
sync::Arc,
};
use hash_db::{Prefix, Hasher};
use sp_trie::{MemoryDB, prefixed_key};
use crate::storage_cache::{new_shared_cache, CachingState, SharedCache};
use hash_db::{Hasher, Prefix};
use kvdb::{DBTransaction, KeyValueDB};
use sp_core::{
hexdisplay::HexDisplay,
storage::{ChildInfo, TrackedStorageKey},
hexdisplay::HexDisplay
};
use sp_runtime::traits::{Block as BlockT, HashFor};
use sp_runtime::Storage;
use sp_runtime::{
traits::{Block as BlockT, HashFor},
Storage,
};
use sp_state_machine::{
DBValue, backend::Backend as StateBackend, StorageCollection, ChildStorageCollection, ProofRecorder,
backend::Backend as StateBackend, ChildStorageCollection, DBValue, ProofRecorder,
StorageCollection,
};
use kvdb::{KeyValueDB, DBTransaction};
use crate::storage_cache::{CachingState, SharedCache, new_shared_cache};
use sp_trie::{prefixed_key, MemoryDB};
type DbState<B> = sp_state_machine::TrieBackend<
Arc<dyn sp_state_machine::Storage<HashFor<B>>>, HashFor<B>
>;
type DbState<B> =
sp_state_machine::TrieBackend<Arc<dyn sp_state_machine::Storage<HashFor<B>>>, HashFor<B>>;
type State<B> = CachingState<DbState<B>, B>;
@@ -53,14 +57,17 @@ impl<Block: BlockT> sp_state_machine::Storage<HashFor<Block>> for StorageDb<Bloc
let prefixed_key = prefixed_key::<HashFor<Block>>(key, prefix);
if let Some(recorder) = &self.proof_recorder {
if let Some(v) = recorder.get(&key) {
return Ok(v.clone());
return Ok(v.clone())
}
let backend_value = self.db.get(0, &prefixed_key)
let backend_value = self
.db
.get(0, &prefixed_key)
.map_err(|e| format!("Database backend error: {:?}", e))?;
recorder.record(key.clone(), backend_value.clone());
Ok(backend_value)
} else {
self.db.get(0, &prefixed_key)
self.db
.get(0, &prefixed_key)
.map_err(|e| format!("Database backend error: {:?}", e))
}
}
@@ -91,7 +98,11 @@ pub struct BenchmarkingState<B: BlockT> {
impl<B: BlockT> BenchmarkingState<B> {
/// Create a new instance that creates a database in a temporary dir.
pub fn new(genesis: Storage, _cache_size_mb: Option<usize>, record_proof: bool) -> Result<Self, String> {
pub fn new(
genesis: Storage,
_cache_size_mb: Option<usize>,
record_proof: bool,
) -> Result<Self, String> {
let mut root = B::Hash::default();
let mut mdb = MemoryDB::<HashFor<B>>::default();
sp_state_machine::TrieDBMut::<HashFor<B>>::new(&mut mdb, &mut root);
@@ -114,14 +125,17 @@ impl<B: BlockT> BenchmarkingState<B> {
state.add_whitelist_to_tracker();
state.reopen()?;
let child_delta = genesis.children_default.iter().map(|(_storage_key, child_content)| (
&child_content.child_info,
child_content.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))),
));
let (root, transaction): (B::Hash, _) = state.state.borrow_mut().as_mut().unwrap().full_storage_root(
genesis.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))),
child_delta,
);
let child_delta = genesis.children_default.iter().map(|(_storage_key, child_content)| {
(
&child_content.child_info,
child_content.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))),
)
});
let (root, transaction): (B::Hash, _) =
state.state.borrow_mut().as_mut().unwrap().full_storage_root(
genesis.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))),
child_delta,
);
state.genesis = transaction.clone().drain();
state.genesis_root = root.clone();
state.commit(root, transaction, Vec::new(), Vec::new())?;
@@ -143,12 +157,12 @@ impl<B: BlockT> BenchmarkingState<B> {
let storage_db = Arc::new(StorageDb::<B> {
db,
proof_recorder: self.proof_recorder.clone(),
_block: Default::default()
_block: Default::default(),
});
*self.state.borrow_mut() = Some(State::new(
DbState::<B>::new(storage_db, self.root.get()),
self.shared_cache.clone(),
None
None,
));
Ok(())
}
@@ -178,7 +192,7 @@ impl<B: BlockT> BenchmarkingState<B> {
let key_tracker = if let Some(childtrie) = childtrie {
child_key_tracker.entry(childtrie.to_vec()).or_insert_with(|| HashMap::new())
} else {
} else {
&mut main_key_tracker
};
@@ -193,7 +207,7 @@ impl<B: BlockT> BenchmarkingState<B> {
let should_log = !tracker.has_been_read();
tracker.add_read();
should_log
}
},
};
if should_log {
@@ -215,7 +229,7 @@ impl<B: BlockT> BenchmarkingState<B> {
let key_tracker = if let Some(childtrie) = childtrie {
child_key_tracker.entry(childtrie.to_vec()).or_insert_with(|| HashMap::new())
} else {
} else {
&mut main_key_tracker
};
@@ -231,7 +245,7 @@ impl<B: BlockT> BenchmarkingState<B> {
let should_log = !tracker.has_been_written();
tracker.add_write();
should_log
}
},
};
if should_log {
@@ -269,7 +283,7 @@ fn state_err() -> String {
}
impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
type Error = <DbState<B> as StateBackend<HashFor<B>>>::Error;
type Error = <DbState<B> as StateBackend<HashFor<B>>>::Error;
type Transaction = <DbState<B> as StateBackend<HashFor<B>>>::Transaction;
type TrieBackendStorage = <DbState<B> as StateBackend<HashFor<B>>>::TrieBackendStorage;
@@ -289,7 +303,11 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
key: &[u8],
) -> Result<Option<Vec<u8>>, Self::Error> {
self.add_read_key(Some(child_info.storage_key()), key);
self.state.borrow().as_ref().ok_or_else(state_err)?.child_storage(child_info, key)
self.state
.borrow()
.as_ref()
.ok_or_else(state_err)?
.child_storage(child_info, key)
}
fn exists_storage(&self, key: &[u8]) -> Result<bool, Self::Error> {
@@ -303,7 +321,11 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
key: &[u8],
) -> Result<bool, Self::Error> {
self.add_read_key(Some(child_info.storage_key()), key);
self.state.borrow().as_ref().ok_or_else(state_err)?.exists_child_storage(child_info, key)
self.state
.borrow()
.as_ref()
.ok_or_else(state_err)?
.exists_child_storage(child_info, key)
}
fn next_storage_key(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
@@ -317,7 +339,11 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
key: &[u8],
) -> Result<Option<Vec<u8>>, Self::Error> {
self.add_read_key(Some(child_info.storage_key()), key);
self.state.borrow().as_ref().ok_or_else(state_err)?.next_child_storage_key(child_info, key)
self.state
.borrow()
.as_ref()
.ok_or_else(state_err)?
.next_child_storage_key(child_info, key)
}
fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
@@ -340,8 +366,13 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
f: F,
allow_missing: bool,
) -> Result<bool, Self::Error> {
self.state.borrow().as_ref().ok_or_else(state_err)?
.apply_to_key_values_while(child_info, prefix, start_at, f, allow_missing)
self.state.borrow().as_ref().ok_or_else(state_err)?.apply_to_key_values_while(
child_info,
prefix,
start_at,
f,
allow_missing,
)
}
fn apply_to_keys_while<F: FnMut(&[u8]) -> bool>(
@@ -368,17 +399,29 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
fn storage_root<'a>(
&self,
delta: impl Iterator<Item=(&'a [u8], Option<&'a [u8]>)>,
) -> (B::Hash, Self::Transaction) where B::Hash: Ord {
self.state.borrow().as_ref().map_or(Default::default(), |s| s.storage_root(delta))
delta: impl Iterator<Item = (&'a [u8], Option<&'a [u8]>)>,
) -> (B::Hash, Self::Transaction)
where
B::Hash: Ord,
{
self.state
.borrow()
.as_ref()
.map_or(Default::default(), |s| s.storage_root(delta))
}
fn child_storage_root<'a>(
&self,
child_info: &ChildInfo,
delta: impl Iterator<Item=(&'a [u8], Option<&'a [u8]>)>,
) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord {
self.state.borrow().as_ref().map_or(Default::default(), |s| s.child_storage_root(child_info, delta))
delta: impl Iterator<Item = (&'a [u8], Option<&'a [u8]>)>,
) -> (B::Hash, bool, Self::Transaction)
where
B::Hash: Ord,
{
self.state
.borrow()
.as_ref()
.map_or(Default::default(), |s| s.child_storage_root(child_info, delta))
}
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
@@ -389,17 +432,16 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
self.state.borrow().as_ref().map_or(Default::default(), |s| s.keys(prefix))
}
fn child_keys(
&self,
child_info: &ChildInfo,
prefix: &[u8],
) -> Vec<Vec<u8>> {
self.state.borrow().as_ref().map_or(Default::default(), |s| s.child_keys(child_info, prefix))
fn child_keys(&self, child_info: &ChildInfo, prefix: &[u8]) -> Vec<Vec<u8>> {
self.state
.borrow()
.as_ref()
.map_or(Default::default(), |s| s.child_keys(child_info, prefix))
}
fn as_trie_backend(&mut self)
-> Option<&sp_state_machine::TrieBackend<Self::TrieBackendStorage, HashFor<B>>>
{
fn as_trie_backend(
&mut self,
) -> Option<&sp_state_machine::TrieBackend<Self::TrieBackendStorage, HashFor<B>>> {
None
}
@@ -425,7 +467,8 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
let mut record = self.record.take();
record.extend(keys);
self.record.set(record);
db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?;
db.write(db_transaction)
.map_err(|_| String::from("Error committing transaction"))?;
self.root.set(storage_root);
self.db.set(Some(db));
@@ -455,7 +498,8 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
None => db_transaction.delete(0, &key),
}
}
db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?;
db.write(db_transaction)
.map_err(|_| String::from("Error committing transaction"))?;
self.db.set(Some(db));
}
@@ -519,24 +563,20 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
let reads = tracker.reads.min(1);
let writes = tracker.writes.min(1);
if let Some(prefix_tracker) = prefix_key_tracker.get_mut(&prefix) {
prefix_tracker.0 += reads;
prefix_tracker.1 += writes;
prefix_tracker.0 += reads;
prefix_tracker.1 += writes;
} else {
prefix_key_tracker.insert(
prefix,
(
reads,
writes,
tracker.whitelisted,
),
);
prefix_key_tracker.insert(prefix, (reads, writes, tracker.whitelisted));
}
}
});
prefix_key_tracker.iter().map(|(key, tracker)| -> (Vec<u8>, u32, u32, bool) {
prefix_key_tracker
.iter()
.map(|(key, tracker)| -> (Vec<u8>, u32, u32, bool) {
(key.to_vec(), tracker.0, tracker.1, tracker.2)
}).collect::<Vec<_>>()
})
.collect::<Vec<_>>()
}
fn register_overlay_stats(&self, stats: &sp_state_machine::StateMachineStats) {
@@ -544,7 +584,10 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
}
fn usage_info(&self) -> sp_state_machine::UsageInfo {
self.state.borrow().as_ref().map_or(sp_state_machine::UsageInfo::empty(), |s| s.usage_info())
self.state
.borrow()
.as_ref()
.map_or(sp_state_machine::UsageInfo::empty(), |s| s.usage_info())
}
fn proof_size(&self) -> Option<u32> {
@@ -585,8 +628,8 @@ mod test {
#[test]
fn read_to_main_and_child_tries() {
let bench_state = BenchmarkingState::<crate::tests::Block>::new(Default::default(), None, false)
.unwrap();
let bench_state =
BenchmarkingState::<crate::tests::Block>::new(Default::default(), None, false).unwrap();
for _ in 0..2 {
let child1 = sp_core::storage::ChildInfo::new_default(b"child1");
@@ -600,16 +643,14 @@ mod test {
bench_state.child_storage(&child1, b"bar").unwrap();
bench_state.child_storage(&child2, b"bar").unwrap();
bench_state.commit(
Default::default(),
Default::default(),
vec![
("foo".as_bytes().to_vec(), None)
],
vec![
("child1".as_bytes().to_vec(), vec![("foo".as_bytes().to_vec(), None)])
]
).unwrap();
bench_state
.commit(
Default::default(),
Default::default(),
vec![("foo".as_bytes().to_vec(), None)],
vec![("child1".as_bytes().to_vec(), vec![("foo".as_bytes().to_vec(), None)])],
)
.unwrap();
let rw_tracker = bench_state.read_write_count();
assert_eq!(rw_tracker.0, 6);
File diff suppressed because it is too large Load Diff
+54 -34
View File
@@ -18,12 +18,11 @@
//! List-cache storage entries.
use codec::{Decode, Encode};
use sp_blockchain::Result as ClientResult;
use sp_runtime::traits::{Block as BlockT, NumberFor};
use codec::{Encode, Decode};
use crate::cache::{CacheItemT, ComplexBlockId};
use crate::cache::list_storage::{Storage};
use crate::cache::{list_storage::Storage, CacheItemT, ComplexBlockId};
/// Single list-based cache entry.
#[derive(Debug)]
@@ -52,10 +51,8 @@ impl<Block: BlockT, T: CacheItemT> Entry<Block, T> {
match value {
Some(value) => match self.value == value {
true => None,
false => Some(StorageEntry {
prev_valid_from: Some(self.valid_from.clone()),
value,
}),
false =>
Some(StorageEntry { prev_valid_from: Some(self.valid_from.clone()), value }),
},
None => None,
}
@@ -67,7 +64,8 @@ impl<Block: BlockT, T: CacheItemT> Entry<Block, T> {
storage: &S,
block: NumberFor<Block>,
) -> ClientResult<Option<(ComplexBlockId<Block>, Option<ComplexBlockId<Block>>)>> {
Ok(self.search_best_before(storage, block)?
Ok(self
.search_best_before(storage, block)?
.map(|(entry, next)| (entry.valid_from, next)))
}
@@ -86,14 +84,14 @@ impl<Block: BlockT, T: CacheItemT> Entry<Block, T> {
let mut current = self.valid_from.clone();
if block >= self.valid_from.number {
let value = self.value.clone();
return Ok(Some((Entry { valid_from: current, value }, next)));
return Ok(Some((Entry { valid_from: current, value }, next)))
}
// else - travel back in time
loop {
let entry = storage.require_entry(&current)?;
if block >= current.number {
return Ok(Some((Entry { valid_from: current, value: entry.value }, next)));
return Ok(Some((Entry { valid_from: current, value: entry.value }, next)))
}
next = Some(current);
@@ -108,18 +106,15 @@ impl<Block: BlockT, T: CacheItemT> Entry<Block, T> {
impl<Block: BlockT, T: CacheItemT> StorageEntry<Block, T> {
/// Converts storage entry into an entry, valid from given block.
pub fn into_entry(self, valid_from: ComplexBlockId<Block>) -> Entry<Block, T> {
Entry {
valid_from,
value: self.value,
}
Entry { valid_from, value: self.value }
}
}
#[cfg(test)]
mod tests {
use crate::cache::list_storage::tests::{DummyStorage, FaultyStorage};
use substrate_test_runtime_client::runtime::{H256, Block};
use super::*;
use crate::cache::list_storage::tests::{DummyStorage, FaultyStorage};
use substrate_test_runtime_client::runtime::{Block, H256};
fn test_id(number: u64) -> ComplexBlockId<Block> {
ComplexBlockId::new(H256::from_low_u64_be(number), number)
@@ -132,36 +127,61 @@ mod tests {
// when trying to update with the same Some value
assert_eq!(Entry { valid_from: test_id(1), value: 1 }.try_update(Some(1)), None);
// when trying to update with different Some value
assert_eq!(Entry { valid_from: test_id(1), value: 1 }.try_update(Some(2)),
Some(StorageEntry { prev_valid_from: Some(test_id(1)), value: 2 }));
assert_eq!(
Entry { valid_from: test_id(1), value: 1 }.try_update(Some(2)),
Some(StorageEntry { prev_valid_from: Some(test_id(1)), value: 2 })
);
}
#[test]
fn entry_search_best_before_fails() {
// when storage returns error
assert!(Entry::<_, u64> { valid_from: test_id(100), value: 42 }
.search_best_before(&FaultyStorage, 50).is_err());
.search_best_before(&FaultyStorage, 50)
.is_err());
}
#[test]
fn entry_search_best_before_works() {
// when block is better than our best block
assert_eq!(Entry::<_, u64> { valid_from: test_id(100), value: 100 }
.search_best_before(&DummyStorage::new(), 150).unwrap(),
Some((Entry::<_, u64> { valid_from: test_id(100), value: 100 }, None)));
assert_eq!(
Entry::<_, u64> { valid_from: test_id(100), value: 100 }
.search_best_before(&DummyStorage::new(), 150)
.unwrap(),
Some((Entry::<_, u64> { valid_from: test_id(100), value: 100 }, None))
);
// when block is found between two entries
assert_eq!(Entry::<_, u64> { valid_from: test_id(100), value: 100 }
.search_best_before(&DummyStorage::new()
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 })
.with_entry(test_id(50), StorageEntry { prev_valid_from: Some(test_id(30)), value: 50 }),
75).unwrap(),
Some((Entry::<_, u64> { valid_from: test_id(50), value: 50 }, Some(test_id(100)))));
assert_eq!(
Entry::<_, u64> { valid_from: test_id(100), value: 100 }
.search_best_before(
&DummyStorage::new()
.with_entry(
test_id(100),
StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 }
)
.with_entry(
test_id(50),
StorageEntry { prev_valid_from: Some(test_id(30)), value: 50 }
),
75
)
.unwrap(),
Some((Entry::<_, u64> { valid_from: test_id(50), value: 50 }, Some(test_id(100))))
);
// when block is not found
assert_eq!(Entry::<_, u64> { valid_from: test_id(100), value: 100 }
.search_best_before(&DummyStorage::new()
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 })
.with_entry(test_id(50), StorageEntry { prev_valid_from: None, value: 50 }),
30).unwrap(),
None);
assert_eq!(
Entry::<_, u64> { valid_from: test_id(100), value: 100 }
.search_best_before(
&DummyStorage::new()
.with_entry(
test_id(100),
StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 }
)
.with_entry(test_id(50), StorageEntry { prev_valid_from: None, value: 50 }),
30
)
.unwrap(),
None
);
}
}
+100 -45
View File
@@ -20,17 +20,23 @@
use std::sync::Arc;
use sp_blockchain::{Error as ClientError, Result as ClientResult};
use codec::{Encode, Decode};
use sp_runtime::generic::BlockId;
use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor};
use sp_database::{Database, Transaction};
use crate::utils::{self, meta_keys};
use codec::{Decode, Encode};
use sp_blockchain::{Error as ClientError, Result as ClientResult};
use sp_database::{Database, Transaction};
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, Header as HeaderT, NumberFor},
};
use crate::cache::{CacheItemT, ComplexBlockId};
use crate::cache::list_cache::{CommitOperation, Fork};
use crate::cache::list_entry::{Entry, StorageEntry};
use crate::DbHash;
use crate::{
cache::{
list_cache::{CommitOperation, Fork},
list_entry::{Entry, StorageEntry},
CacheItemT, ComplexBlockId,
},
DbHash,
};
/// Single list-cache metadata.
#[derive(Debug)]
@@ -54,14 +60,21 @@ pub trait Storage<Block: BlockT, T: CacheItemT> {
fn read_meta(&self) -> ClientResult<Metadata<Block>>;
/// Reads cache entry from the storage.
fn read_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>>;
fn read_entry(
&self,
at: &ComplexBlockId<Block>,
) -> ClientResult<Option<StorageEntry<Block, T>>>;
/// Reads referenced (and thus existing) cache entry from the storage.
fn require_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<StorageEntry<Block, T>> {
self.read_entry(at)
.and_then(|entry| entry
.ok_or_else(|| ClientError::from(
ClientError::Backend(format!("Referenced cache entry at {:?} is not found", at)))))
self.read_entry(at).and_then(|entry| {
entry.ok_or_else(|| {
ClientError::from(ClientError::Backend(format!(
"Referenced cache entry at {:?} is not found",
at
)))
})
})
}
}
@@ -111,10 +124,14 @@ impl DbStorage {
}
/// Get reference to the database.
pub fn db(&self) -> &Arc<dyn Database<DbHash>> { &self.db }
pub fn db(&self) -> &Arc<dyn Database<DbHash>> {
&self.db
}
/// Get reference to the database columns.
pub fn columns(&self) -> &DbColumns { &self.columns }
pub fn columns(&self) -> &DbColumns {
&self.columns
}
/// Encode block id for storing as a key in cache column.
/// We append prefix to the actual encoding to allow several caches
@@ -128,25 +145,35 @@ impl DbStorage {
impl<Block: BlockT, T: CacheItemT> Storage<Block, T> for DbStorage {
fn read_id(&self, at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>> {
utils::read_header::<Block>(&*self.db, self.columns.key_lookup, self.columns.header, BlockId::Number(at))
.map(|maybe_header| maybe_header.map(|header| header.hash()))
utils::read_header::<Block>(
&*self.db,
self.columns.key_lookup,
self.columns.header,
BlockId::Number(at),
)
.map(|maybe_header| maybe_header.map(|header| header.hash()))
}
fn read_header(&self, at: &Block::Hash) -> ClientResult<Option<Block::Header>> {
utils::read_header::<Block>(&*self.db, self.columns.key_lookup, self.columns.header, BlockId::Hash(*at))
utils::read_header::<Block>(
&*self.db,
self.columns.key_lookup,
self.columns.header,
BlockId::Hash(*at),
)
}
fn read_meta(&self) -> ClientResult<Metadata<Block>> {
match self.db.get(self.columns.meta, &self.meta_key) {
Some(meta) => meta::decode(&*meta),
None => Ok(Metadata {
finalized: None,
unfinalized: Vec::new(),
})
None => Ok(Metadata { finalized: None, unfinalized: Vec::new() }),
}
}
fn read_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>> {
fn read_entry(
&self,
at: &ComplexBlockId<Block>,
) -> ClientResult<Option<StorageEntry<Block, T>>> {
match self.db.get(self.columns.cache, &self.encode_block_id(at)) {
Some(entry) => StorageEntry::<Block, T>::decode(&mut &entry[..])
.map_err(|_| ClientError::Backend("Failed to decode cache entry".into()))
@@ -171,7 +198,11 @@ impl<'a> DbStorageTransaction<'a> {
impl<'a, Block: BlockT, T: CacheItemT> StorageTransaction<Block, T> for DbStorageTransaction<'a> {
fn insert_storage_entry(&mut self, at: &ComplexBlockId<Block>, entry: &StorageEntry<Block, T>) {
self.tx.set_from_vec(self.storage.columns.cache, &self.storage.encode_block_id(at), entry.encode());
self.tx.set_from_vec(
self.storage.columns.cache,
&self.storage.encode_block_id(at),
entry.encode(),
);
}
fn remove_storage_entry(&mut self, at: &ComplexBlockId<Block>) {
@@ -187,7 +218,8 @@ impl<'a, Block: BlockT, T: CacheItemT> StorageTransaction<Block, T> for DbStorag
self.tx.set_from_vec(
self.storage.columns.meta,
&self.storage.meta_key,
meta::encode(best_finalized_entry, unfinalized, operation));
meta::encode(best_finalized_entry, unfinalized, operation),
);
}
}
@@ -206,10 +238,11 @@ mod meta {
pub fn encode<Block: BlockT, T: CacheItemT>(
best_finalized_entry: Option<&Entry<Block, T>>,
unfinalized: &[Fork<Block, T>],
op: &CommitOperation<Block, T>
op: &CommitOperation<Block, T>,
) -> Vec<u8> {
let mut finalized = best_finalized_entry.as_ref().map(|entry| &entry.valid_from);
let mut unfinalized = unfinalized.iter().map(|fork| &fork.head().valid_from).collect::<Vec<_>>();
let mut unfinalized =
unfinalized.iter().map(|fork| &fork.head().valid_from).collect::<Vec<_>>();
match op {
CommitOperation::AppendNewBlock(_, _) => (),
@@ -230,8 +263,11 @@ mod meta {
CommitOperation::BlockReverted(ref forks) => {
for (fork_index, updated_fork) in forks.iter().rev() {
match updated_fork {
Some(updated_fork) => unfinalized[*fork_index] = &updated_fork.head().valid_from,
None => { unfinalized.remove(*fork_index); },
Some(updated_fork) =>
unfinalized[*fork_index] = &updated_fork.head().valid_from,
None => {
unfinalized.remove(*fork_index);
},
}
}
},
@@ -243,10 +279,12 @@ mod meta {
/// Decode meta information.
pub fn decode<Block: BlockT>(encoded: &[u8]) -> ClientResult<Metadata<Block>> {
let input = &mut &*encoded;
let finalized: Option<ComplexBlockId<Block>> = Decode::decode(input)
.map_err(|_| ClientError::from(ClientError::Backend("Error decoding cache meta".into())))?;
let unfinalized: Vec<ComplexBlockId<Block>> = Decode::decode(input)
.map_err(|_| ClientError::from(ClientError::Backend("Error decoding cache meta".into())))?;
let finalized: Option<ComplexBlockId<Block>> = Decode::decode(input).map_err(|_| {
ClientError::from(ClientError::Backend("Error decoding cache meta".into()))
})?;
let unfinalized: Vec<ComplexBlockId<Block>> = Decode::decode(input).map_err(|_| {
ClientError::from(ClientError::Backend("Error decoding cache meta".into()))
})?;
Ok(Metadata { finalized, unfinalized })
}
@@ -254,8 +292,8 @@ mod meta {
#[cfg(test)]
pub mod tests {
use std::collections::{HashMap, HashSet};
use super::*;
use std::collections::{HashMap, HashSet};
pub struct FaultyStorage;
@@ -272,7 +310,10 @@ pub mod tests {
Err(ClientError::Backend("TestError".into()))
}
fn read_entry(&self, _at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>> {
fn read_entry(
&self,
_at: &ComplexBlockId<Block>,
) -> ClientResult<Option<StorageEntry<Block, T>>> {
Err(ClientError::Backend("TestError".into()))
}
}
@@ -287,17 +328,18 @@ pub mod tests {
impl<Block: BlockT, T: CacheItemT> DummyStorage<Block, T> {
pub fn new() -> Self {
DummyStorage {
meta: Metadata {
finalized: None,
unfinalized: Vec::new(),
},
meta: Metadata { finalized: None, unfinalized: Vec::new() },
ids: HashMap::new(),
headers: HashMap::new(),
entries: HashMap::new(),
}
}
pub fn with_meta(mut self, finalized: Option<ComplexBlockId<Block>>, unfinalized: Vec<ComplexBlockId<Block>>) -> Self {
pub fn with_meta(
mut self,
finalized: Option<ComplexBlockId<Block>>,
unfinalized: Vec<ComplexBlockId<Block>>,
) -> Self {
self.meta.finalized = finalized;
self.meta.unfinalized = unfinalized;
self
@@ -313,7 +355,11 @@ pub mod tests {
self
}
pub fn with_entry(mut self, at: ComplexBlockId<Block>, entry: StorageEntry<Block, T>) -> Self {
pub fn with_entry(
mut self,
at: ComplexBlockId<Block>,
entry: StorageEntry<Block, T>,
) -> Self {
self.entries.insert(at.hash, entry);
self
}
@@ -332,7 +378,10 @@ pub mod tests {
Ok(self.meta.clone())
}
fn read_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>> {
fn read_entry(
&self,
at: &ComplexBlockId<Block>,
) -> ClientResult<Option<StorageEntry<Block, T>>> {
Ok(self.entries.get(&at.hash).cloned())
}
}
@@ -366,7 +415,11 @@ pub mod tests {
}
impl<Block: BlockT, T: CacheItemT> StorageTransaction<Block, T> for DummyTransaction<Block> {
fn insert_storage_entry(&mut self, at: &ComplexBlockId<Block>, _entry: &StorageEntry<Block, T>) {
fn insert_storage_entry(
&mut self,
at: &ComplexBlockId<Block>,
_entry: &StorageEntry<Block, T>,
) {
self.inserted_entries.insert(at.hash);
}
@@ -380,7 +433,9 @@ pub mod tests {
unfinalized: &[Fork<Block, T>],
operation: &CommitOperation<Block, T>,
) {
self.updated_meta = Some(meta::decode(&meta::encode(best_finalized_entry, unfinalized, operation)).unwrap());
self.updated_meta = Some(
meta::decode(&meta::encode(best_finalized_entry, unfinalized, operation)).unwrap(),
);
}
}
}
+63 -59
View File
@@ -18,17 +18,27 @@
//! DB-backed cache of blockchain data.
use std::{sync::Arc, collections::{HashMap, hash_map::Entry}};
use parking_lot::RwLock;
use std::{
collections::{hash_map::Entry, HashMap},
sync::Arc,
};
use sc_client_api::blockchain::{well_known_cache_keys::{self, Id as CacheKeyId}, Cache as BlockchainCache};
use sp_blockchain::{Result as ClientResult, HeaderMetadataCache};
use crate::{
utils::{self, COLUMN_META},
DbHash,
};
use codec::{Decode, Encode};
use sc_client_api::blockchain::{
well_known_cache_keys::{self, Id as CacheKeyId},
Cache as BlockchainCache,
};
use sp_blockchain::{HeaderMetadataCache, Result as ClientResult};
use sp_database::{Database, Transaction};
use codec::{Encode, Decode};
use sp_runtime::generic::BlockId;
use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero};
use crate::utils::{self, COLUMN_META};
use crate::DbHash;
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero},
};
use self::list_cache::{ListCache, PruningStrategy};
@@ -118,7 +128,10 @@ impl<Block: BlockT> DbCache<Block> {
}
/// Begin cache transaction.
pub fn transaction<'a>(&'a mut self, tx: &'a mut Transaction<DbHash>) -> DbCacheTransaction<'a, Block> {
pub fn transaction<'a>(
&'a mut self,
tx: &'a mut Transaction<DbHash>,
) -> DbCacheTransaction<'a, Block> {
DbCacheTransaction {
cache: self,
tx,
@@ -164,7 +177,7 @@ impl<Block: BlockT> DbCache<Block> {
self.key_lookup_column,
self.header_column,
self.cache_column,
&self.best_finalized_block
&self.best_finalized_block,
)
}
}
@@ -184,19 +197,16 @@ fn get_cache_helper<'a, Block: BlockT>(
Entry::Occupied(entry) => Ok(entry.into_mut()),
Entry::Vacant(entry) => {
let cache = ListCache::new(
self::list_storage::DbStorage::new(name.to_vec(), db.clone(),
self::list_storage::DbColumns {
meta: COLUMN_META,
key_lookup,
header,
cache,
},
self::list_storage::DbStorage::new(
name.to_vec(),
db.clone(),
self::list_storage::DbColumns { meta: COLUMN_META, key_lookup, header, cache },
),
cache_pruning_strategy(name),
best_finalized_block.clone(),
)?;
Ok(entry.insert(cache))
}
},
}
}
@@ -210,10 +220,7 @@ pub struct DbCacheTransactionOps<Block: BlockT> {
impl<Block: BlockT> DbCacheTransactionOps<Block> {
/// Empty transaction ops.
pub fn empty() -> DbCacheTransactionOps<Block> {
DbCacheTransactionOps {
cache_at_ops: HashMap::new(),
best_finalized_block: None,
}
DbCacheTransactionOps { cache_at_ops: HashMap::new(), best_finalized_block: None }
}
}
@@ -244,19 +251,21 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
) -> ClientResult<Self> {
// prepare list of caches that are not update
// (we might still need to do some cache maintenance in this case)
let missed_caches = self.cache.cache_at.keys()
let missed_caches = self
.cache
.cache_at
.keys()
.filter(|cache| !data_at.contains_key(*cache))
.cloned()
.collect::<Vec<_>>();
let mut insert_op = |name: CacheKeyId, value: Option<Vec<u8>>| -> Result<(), sp_blockchain::Error> {
let mut insert_op = |name: CacheKeyId,
value: Option<Vec<u8>>|
-> Result<(), sp_blockchain::Error> {
let cache = self.cache.get_cache(name)?;
let cache_ops = self.cache_at_ops.entry(name).or_default();
cache.on_block_insert(
&mut self::list_storage::DbStorageTransaction::new(
cache.storage(),
&mut self.tx,
),
&mut self::list_storage::DbStorageTransaction::new(cache.storage(), &mut self.tx),
parent.clone(),
block.clone(),
value,
@@ -271,8 +280,7 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
missed_caches.into_iter().try_for_each(|name| insert_op(name, None))?;
match entry_type {
EntryType::Final | EntryType::Genesis =>
self.best_finalized_block = Some(block),
EntryType::Final | EntryType::Genesis => self.best_finalized_block = Some(block),
EntryType::NonFinal => (),
}
@@ -288,10 +296,7 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
for (name, cache) in self.cache.cache_at.iter() {
let cache_ops = self.cache_at_ops.entry(*name).or_default();
cache.on_block_finalize(
&mut self::list_storage::DbStorageTransaction::new(
cache.storage(),
&mut self.tx
),
&mut self::list_storage::DbStorageTransaction::new(cache.storage(), &mut self.tx),
parent.clone(),
block.clone(),
cache_ops,
@@ -304,17 +309,11 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
}
/// When block is reverted.
pub fn on_block_revert(
mut self,
reverted_block: &ComplexBlockId<Block>,
) -> ClientResult<Self> {
pub fn on_block_revert(mut self, reverted_block: &ComplexBlockId<Block>) -> ClientResult<Self> {
for (name, cache) in self.cache.cache_at.iter() {
let cache_ops = self.cache_at_ops.entry(*name).or_default();
cache.on_block_revert(
&mut self::list_storage::DbStorageTransaction::new(
cache.storage(),
&mut self.tx
),
&mut self::list_storage::DbStorageTransaction::new(cache.storage(), &mut self.tx),
reverted_block,
cache_ops,
)?;
@@ -352,7 +351,9 @@ impl<Block: BlockT> BlockchainCache<Block> for DbCacheSync<Block> {
&self,
key: &CacheKeyId,
at: &BlockId<Block>,
) -> ClientResult<Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)>> {
) -> ClientResult<
Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)>,
> {
let mut cache = self.0.write();
let header_metadata_cache = cache.header_metadata_cache.clone();
let cache = cache.get_cache(*key)?;
@@ -360,36 +361,39 @@ impl<Block: BlockT> BlockchainCache<Block> for DbCacheSync<Block> {
let db = storage.db();
let columns = storage.columns();
let at = match *at {
BlockId::Hash(hash) => {
match header_metadata_cache.header_metadata(hash) {
Some(metadata) => ComplexBlockId::new(hash, metadata.number),
None => {
let header = utils::require_header::<Block>(
&**db,
columns.key_lookup,
columns.header,
BlockId::Hash(hash.clone()))?;
ComplexBlockId::new(hash, *header.number())
}
}
BlockId::Hash(hash) => match header_metadata_cache.header_metadata(hash) {
Some(metadata) => ComplexBlockId::new(hash, metadata.number),
None => {
let header = utils::require_header::<Block>(
&**db,
columns.key_lookup,
columns.header,
BlockId::Hash(hash.clone()),
)?;
ComplexBlockId::new(hash, *header.number())
},
},
BlockId::Number(number) => {
let hash = utils::require_header::<Block>(
&**db,
columns.key_lookup,
columns.header,
BlockId::Number(number.clone()))?.hash();
BlockId::Number(number.clone()),
)?
.hash();
ComplexBlockId::new(hash, number)
},
};
cache.value_at_block(&at)
.map(|block_and_value| block_and_value.map(|(begin_block, end_block, value)|
cache.value_at_block(&at).map(|block_and_value| {
block_and_value.map(|(begin_block, end_block, value)| {
(
(begin_block.number, begin_block.hash),
end_block.map(|end_block| (end_block.number, end_block.hash)),
value,
)))
)
})
})
}
}
+325 -182
View File
@@ -18,33 +18,43 @@
//! DB-backed changes tries storage.
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use hash_db::Prefix;
use codec::{Decode, Encode};
use parking_lot::RwLock;
use sp_blockchain::{Error as ClientError, Result as ClientResult};
use sp_trie::MemoryDB;
use sc_client_api::backend::PrunableStateChangesTrieStorage;
use sp_blockchain::{well_known_cache_keys, Cache as BlockchainCache, HeaderMetadataCache};
use sp_core::{ChangesTrieConfiguration, ChangesTrieConfigurationRange, convert_hash};
use sp_core::storage::PrefixedStorageKey;
use sp_database::Transaction;
use sp_runtime::traits::{
Block as BlockT, Header as HeaderT, HashFor, NumberFor, One, Zero, CheckedSub,
use crate::{
cache::{
ComplexBlockId, DbCache, DbCacheSync, DbCacheTransactionOps, EntryType as CacheEntryType,
},
utils::{self, meta_keys, Meta},
Database, DbHash,
};
use codec::{Decode, Encode};
use hash_db::Prefix;
use parking_lot::RwLock;
use sc_client_api::backend::PrunableStateChangesTrieStorage;
use sp_blockchain::{
well_known_cache_keys, Cache as BlockchainCache, Error as ClientError, HeaderMetadataCache,
Result as ClientResult,
};
use sp_core::{
convert_hash, storage::PrefixedStorageKey, ChangesTrieConfiguration,
ChangesTrieConfigurationRange,
};
use sp_database::Transaction;
use sp_runtime::{
generic::{BlockId, ChangesTrieSignal, DigestItem},
traits::{Block as BlockT, CheckedSub, HashFor, Header as HeaderT, NumberFor, One, Zero},
};
use sp_runtime::generic::{BlockId, DigestItem, ChangesTrieSignal};
use sp_state_machine::{ChangesTrieBuildCache, ChangesTrieCacheAction};
use crate::{Database, DbHash};
use crate::utils::{self, Meta, meta_keys};
use crate::cache::{
DbCacheSync, DbCache, DbCacheTransactionOps,
ComplexBlockId, EntryType as CacheEntryType,
use sp_trie::MemoryDB;
use std::{
collections::{HashMap, HashSet},
sync::Arc,
};
/// Extract new changes trie configuration (if available) from the header.
pub fn extract_new_configuration<Header: HeaderT>(header: &Header) -> Option<&Option<ChangesTrieConfiguration>> {
header.digest()
pub fn extract_new_configuration<Header: HeaderT>(
header: &Header,
) -> Option<&Option<ChangesTrieConfiguration>> {
header
.digest()
.log(DigestItem::as_changes_trie_signal)
.and_then(ChangesTrieSignal::as_new_configuration)
}
@@ -68,10 +78,7 @@ impl<Block: BlockT> DbChangesTrieStorageTransaction<Block> {
impl<Block: BlockT> From<DbCacheTransactionOps<Block>> for DbChangesTrieStorageTransaction<Block> {
fn from(cache_ops: DbCacheTransactionOps<Block>) -> Self {
DbChangesTrieStorageTransaction {
cache_ops,
new_config: None,
}
DbChangesTrieStorageTransaction { cache_ops, new_config: None }
}
}
@@ -173,21 +180,25 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
let new_configuration = match new_configuration {
Some(new_configuration) => new_configuration,
None if !finalized => return Ok(DbCacheTransactionOps::empty().into()),
None => return self.finalize(
tx,
parent_block.hash,
block.hash,
block.number,
Some(new_header),
cache_tx,
),
None =>
return self.finalize(
tx,
parent_block.hash,
block.hash,
block.number,
Some(new_header),
cache_tx,
),
};
// update configuration cache
let mut cache_at = HashMap::new();
cache_at.insert(well_known_cache_keys::CHANGES_TRIE_CONFIG, new_configuration.encode());
Ok(DbChangesTrieStorageTransaction::from(match cache_tx {
Some(cache_tx) => self.cache.0.write()
Some(cache_tx) => self
.cache
.0
.write()
.transaction_with_ops(tx, cache_tx.cache_ops)
.on_block_insert(
parent_block,
@@ -196,7 +207,10 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
if finalized { CacheEntryType::Final } else { CacheEntryType::NonFinal },
)?
.into_ops(),
None => self.cache.0.write()
None => self
.cache
.0
.write()
.transaction(tx)
.on_block_insert(
parent_block,
@@ -205,7 +219,8 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
if finalized { CacheEntryType::Final } else { CacheEntryType::NonFinal },
)?
.into_ops(),
}).with_new_config(Some(new_configuration)))
})
.with_new_config(Some(new_configuration)))
}
/// Called when block is finalized.
@@ -226,7 +241,7 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
if cache_tx.is_some() {
if let Some(new_header) = new_header {
if new_header.hash() == block_hash {
return Ok(cache_tx.expect("guarded by cache_tx.is_some(); qed"));
return Ok(cache_tx.expect("guarded by cache_tx.is_some(); qed"))
}
}
}
@@ -237,22 +252,21 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
let parent_block = ComplexBlockId::new(parent_block_hash, parent_block_num);
Ok(match cache_tx {
Some(cache_tx) => DbChangesTrieStorageTransaction::from(
self.cache.0.write()
self.cache
.0
.write()
.transaction_with_ops(tx, cache_tx.cache_ops)
.on_block_finalize(
parent_block,
block,
)?
.into_ops()
).with_new_config(cache_tx.new_config),
.on_block_finalize(parent_block, block)?
.into_ops(),
)
.with_new_config(cache_tx.new_config),
None => DbChangesTrieStorageTransaction::from(
self.cache.0.write()
self.cache
.0
.write()
.transaction(tx)
.on_block_finalize(
parent_block,
block,
)?
.into_ops()
.on_block_finalize(parent_block, block)?
.into_ops(),
),
})
}
@@ -263,23 +277,24 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
tx: &mut Transaction<DbHash>,
block: &ComplexBlockId<Block>,
) -> ClientResult<DbChangesTrieStorageTransaction<Block>> {
Ok(self.cache.0.write().transaction(tx)
.on_block_revert(block)?
.into_ops()
.into())
Ok(self.cache.0.write().transaction(tx).on_block_revert(block)?.into_ops().into())
}
/// When transaction has been committed.
pub fn post_commit(&self, tx: Option<DbChangesTrieStorageTransaction<Block>>) {
if let Some(tx) = tx {
self.cache.0.write().commit(tx.cache_ops)
.expect("only fails if cache with given name isn't loaded yet;\
cache is already loaded because there is tx; qed");
self.cache.0.write().commit(tx.cache_ops).expect(
"only fails if cache with given name isn't loaded yet;\
cache is already loaded because there is tx; qed",
);
}
}
/// Commit changes into changes trie build cache.
pub fn commit_build_cache(&self, cache_update: ChangesTrieCacheAction<Block::Hash, NumberFor<Block>>) {
pub fn commit_build_cache(
&self,
cache_update: ChangesTrieCacheAction<Block::Hash, NumberFor<Block>>,
) {
self.build_cache.write().perform(cache_update);
}
@@ -307,7 +322,7 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
// 2) or we are (or were) in period where changes tries are disabled
if let Some((begin, end)) = tries_meta.oldest_digest_range {
if block_num <= end || block_num - end <= min_blocks_to_keep.into() {
break;
break
}
tries_meta.oldest_pruned_digest_range_end = end;
@@ -333,7 +348,8 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
self.key_lookup_column,
self.header_column,
BlockId::Number(next_digest_range_start),
)?.hash(),
)?
.hash(),
};
let config_for_new_block = new_header
@@ -341,21 +357,18 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
.unwrap_or(false);
let next_config = match cache_tx {
Some(cache_tx) if config_for_new_block && cache_tx.new_config.is_some() => {
let config = cache_tx
.new_config
.clone()
.expect("guarded by is_some(); qed");
let config = cache_tx.new_config.clone().expect("guarded by is_some(); qed");
ChangesTrieConfigurationRange {
zero: (block_num, block_hash),
end: None,
config,
}
},
_ if config_for_new_block => {
self.configuration_at(&BlockId::Hash(*new_header.expect(
"config_for_new_block is only true when new_header is passed; qed"
).parent_hash()))?
},
_ if config_for_new_block => self.configuration_at(&BlockId::Hash(
*new_header
.expect("config_for_new_block is only true when new_header is passed; qed")
.parent_hash(),
))?,
_ => self.configuration_at(&BlockId::Hash(next_digest_range_start_hash))?,
};
if let Some(config) = next_config.config {
@@ -370,11 +383,11 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
}
tries_meta.oldest_digest_range = Some(oldest_digest_range);
continue;
continue
}
tries_meta.oldest_digest_range = None;
break;
break
}
write_tries_meta(tx, self.meta_column, &*tries_meta);
@@ -383,17 +396,23 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
}
impl<Block: BlockT> PrunableStateChangesTrieStorage<Block> for DbChangesTrieStorage<Block> {
fn storage(&self) -> &dyn sp_state_machine::ChangesTrieStorage<HashFor<Block>, NumberFor<Block>> {
fn storage(
&self,
) -> &dyn sp_state_machine::ChangesTrieStorage<HashFor<Block>, NumberFor<Block>> {
self
}
fn configuration_at(&self, at: &BlockId<Block>) -> ClientResult<
ChangesTrieConfigurationRange<NumberFor<Block>, Block::Hash>
> {
fn configuration_at(
&self,
at: &BlockId<Block>,
) -> ClientResult<ChangesTrieConfigurationRange<NumberFor<Block>, Block::Hash>> {
self.cache
.get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, at)?
.and_then(|(zero, end, encoded)| Decode::decode(&mut &encoded[..]).ok()
.map(|config| ChangesTrieConfigurationRange { zero, end, config }))
.and_then(|(zero, end, encoded)| {
Decode::decode(&mut &encoded[..])
.ok()
.map(|config| ChangesTrieConfigurationRange { zero, end, config })
})
.ok_or_else(|| ClientError::ErrorReadingChangesTriesConfig)
}
@@ -409,14 +428,21 @@ impl<Block: BlockT> sp_state_machine::ChangesTrieRootsStorage<HashFor<Block>, Nu
&self,
hash: Block::Hash,
) -> Result<sp_state_machine::ChangesTrieAnchorBlockId<Block::Hash, NumberFor<Block>>, String> {
utils::read_header::<Block>(&*self.db, self.key_lookup_column, self.header_column, BlockId::Hash(hash))
.map_err(|e| e.to_string())
.and_then(|maybe_header| maybe_header.map(|header|
sp_state_machine::ChangesTrieAnchorBlockId {
utils::read_header::<Block>(
&*self.db,
self.key_lookup_column,
self.header_column,
BlockId::Hash(hash),
)
.map_err(|e| e.to_string())
.and_then(|maybe_header| {
maybe_header
.map(|header| sp_state_machine::ChangesTrieAnchorBlockId {
hash,
number: *header.number(),
}
).ok_or_else(|| format!("Unknown header: {}", hash)))
})
.ok_or_else(|| format!("Unknown header: {}", hash))
})
}
fn root(
@@ -426,7 +452,10 @@ impl<Block: BlockT> sp_state_machine::ChangesTrieRootsStorage<HashFor<Block>, Nu
) -> Result<Option<Block::Hash>, String> {
// check API requirement: we can't get NEXT block(s) based on anchor
if block > anchor.number {
return Err(format!("Can't get changes trie root at {} using anchor at {}", block, anchor.number));
return Err(format!(
"Can't get changes trie root at {} using anchor at {}",
block, anchor.number
))
}
// we need to get hash of the block to resolve changes trie root
@@ -438,8 +467,12 @@ impl<Block: BlockT> sp_state_machine::ChangesTrieRootsStorage<HashFor<Block>, Nu
let mut current_num = anchor.number;
let mut current_hash: Block::Hash = convert_hash(&anchor.hash);
let maybe_anchor_header: Block::Header = utils::require_header::<Block>(
&*self.db, self.key_lookup_column, self.header_column, BlockId::Number(current_num)
).map_err(|e| e.to_string())?;
&*self.db,
self.key_lookup_column,
self.header_column,
BlockId::Number(current_num),
)
.map_err(|e| e.to_string())?;
if maybe_anchor_header.hash() == current_hash {
// if anchor is canonicalized, then the block is also canonicalized
BlockId::Number(block)
@@ -449,8 +482,12 @@ impl<Block: BlockT> sp_state_machine::ChangesTrieRootsStorage<HashFor<Block>, Nu
// back from the anchor to the block with given number
while current_num != block {
let current_header: Block::Header = utils::require_header::<Block>(
&*self.db, self.key_lookup_column, self.header_column, BlockId::Hash(current_hash)
).map_err(|e| e.to_string())?;
&*self.db,
self.key_lookup_column,
self.header_column,
BlockId::Hash(current_hash),
)
.map_err(|e| e.to_string())?;
current_hash = *current_header.parent_hash();
current_num = current_num - One::one();
@@ -460,18 +497,16 @@ impl<Block: BlockT> sp_state_machine::ChangesTrieRootsStorage<HashFor<Block>, Nu
}
};
Ok(
utils::require_header::<Block>(
&*self.db,
self.key_lookup_column,
self.header_column,
block_id,
)
.map_err(|e| e.to_string())?
.digest()
.log(DigestItem::as_changes_trie_root)
.cloned()
Ok(utils::require_header::<Block>(
&*self.db,
self.key_lookup_column,
self.header_column,
block_id,
)
.map_err(|e| e.to_string())?
.digest()
.log(DigestItem::as_changes_trie_root)
.cloned())
}
}
@@ -480,7 +515,9 @@ impl<Block> sp_state_machine::ChangesTrieStorage<HashFor<Block>, NumberFor<Block
where
Block: BlockT,
{
fn as_roots_storage(&self) -> &dyn sp_state_machine::ChangesTrieRootsStorage<HashFor<Block>, NumberFor<Block>> {
fn as_roots_storage(
&self,
) -> &dyn sp_state_machine::ChangesTrieRootsStorage<HashFor<Block>, NumberFor<Block>> {
self
}
@@ -503,8 +540,9 @@ fn read_tries_meta<Block: BlockT>(
meta_column: u32,
) -> ClientResult<ChangesTriesMeta<Block>> {
match db.get(meta_column, meta_keys::CHANGES_TRIES_META) {
Some(h) => Decode::decode(&mut &h[..])
.map_err(|err| ClientError::Backend(format!("Error decoding changes tries metadata: {}", err))),
Some(h) => Decode::decode(&mut &h[..]).map_err(|err| {
ClientError::Backend(format!("Error decoding changes tries metadata: {}", err))
}),
None => Ok(ChangesTriesMeta {
oldest_digest_range: None,
oldest_pruned_digest_range_end: Zero::zero(),
@@ -523,18 +561,23 @@ fn write_tries_meta<Block: BlockT>(
#[cfg(test)]
mod tests {
use super::*;
use crate::{
tests::{insert_header, prepare_changes, Block},
Backend,
};
use hash_db::EMPTY_PREFIX;
use sc_client_api::backend::{
Backend as ClientBackend, NewBlockState, BlockImportOperation, PrunableStateChangesTrieStorage,
Backend as ClientBackend, BlockImportOperation, NewBlockState,
PrunableStateChangesTrieStorage,
};
use sp_blockchain::HeaderBackend as BlockchainHeaderBackend;
use sp_core::H256;
use sp_runtime::testing::{Digest, Header};
use sp_runtime::traits::{Hash, BlakeTwo256};
use sp_runtime::{
testing::{Digest, Header},
traits::{BlakeTwo256, Hash},
};
use sp_state_machine::{ChangesTrieRootsStorage, ChangesTrieStorage};
use crate::Backend;
use crate::tests::{Block, insert_header, prepare_changes};
use super::*;
fn changes(number: u64) -> Option<Vec<(Vec<u8>, Vec<u8>)>> {
Some(vec![(number.to_le_bytes().to_vec(), number.to_le_bytes().to_vec())])
@@ -554,7 +597,9 @@ mod tests {
digest.push(DigestItem::ChangesTrieRoot(root));
changes_trie_update = update;
}
digest.push(DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(new_configuration)));
digest.push(DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(
new_configuration,
)));
let header = Header {
number,
@@ -573,7 +618,8 @@ mod tests {
let mut op = backend.begin_operation().unwrap();
backend.begin_state_operation(&mut op, block_id).unwrap();
op.set_block_data(header, None, None, None, NewBlockState::Best).unwrap();
op.update_changes_trie((changes_trie_update, ChangesTrieCacheAction::Clear)).unwrap();
op.update_changes_trie((changes_trie_update, ChangesTrieCacheAction::Clear))
.unwrap();
backend.commit_operation(op).unwrap();
header_hash
@@ -584,11 +630,13 @@ mod tests {
let backend = Backend::<Block>::new_test(1000, 100);
backend.changes_tries_storage.meta.write().finalized_number = 1000;
let check_changes = |backend: &Backend<Block>, block: u64, changes: Vec<(Vec<u8>, Vec<u8>)>| {
let check_changes = |backend: &Backend<Block>,
block: u64,
changes: Vec<(Vec<u8>, Vec<u8>)>| {
let (changes_root, mut changes_trie_update) = prepare_changes(changes);
let anchor = sp_state_machine::ChangesTrieAnchorBlockId {
hash: backend.blockchain().header(BlockId::Number(block)).unwrap().unwrap().hash(),
number: block
number: block,
};
assert_eq!(backend.changes_tries_storage.root(&anchor, block), Ok(Some(changes_root)));
@@ -605,7 +653,13 @@ mod tests {
];
let changes2 = vec![(b"key_at_2".to_vec(), b"val_at_2".to_vec())];
let block0 = insert_header(&backend, 0, Default::default(), Some(changes0.clone()), Default::default());
let block0 = insert_header(
&backend,
0,
Default::default(),
Some(changes0.clone()),
Default::default(),
);
let block1 = insert_header(&backend, 1, block0, Some(changes1.clone()), Default::default());
let _ = insert_header(&backend, 2, block1, Some(changes2.clone()), Default::default());
@@ -622,19 +676,29 @@ mod tests {
let changes0 = vec![(b"k0".to_vec(), b"v0".to_vec())];
let changes1 = vec![(b"k1".to_vec(), b"v1".to_vec())];
let changes2 = vec![(b"k2".to_vec(), b"v2".to_vec())];
let block0 = insert_header(&backend, 0, Default::default(), Some(changes0.clone()), Default::default());
let block0 = insert_header(
&backend,
0,
Default::default(),
Some(changes0.clone()),
Default::default(),
);
let block1 = insert_header(&backend, 1, block0, Some(changes1.clone()), Default::default());
let block2 = insert_header(&backend, 2, block1, Some(changes2.clone()), Default::default());
let changes2_1_0 = vec![(b"k3".to_vec(), b"v3".to_vec())];
let changes2_1_1 = vec![(b"k4".to_vec(), b"v4".to_vec())];
let block2_1_0 = insert_header(&backend, 3, block2, Some(changes2_1_0.clone()), Default::default());
let block2_1_1 = insert_header(&backend, 4, block2_1_0, Some(changes2_1_1.clone()), Default::default());
let block2_1_0 =
insert_header(&backend, 3, block2, Some(changes2_1_0.clone()), Default::default());
let block2_1_1 =
insert_header(&backend, 4, block2_1_0, Some(changes2_1_1.clone()), Default::default());
let changes2_2_0 = vec![(b"k5".to_vec(), b"v5".to_vec())];
let changes2_2_1 = vec![(b"k6".to_vec(), b"v6".to_vec())];
let block2_2_0 = insert_header(&backend, 3, block2, Some(changes2_2_0.clone()), Default::default());
let block2_2_1 = insert_header(&backend, 4, block2_2_0, Some(changes2_2_1.clone()), Default::default());
let block2_2_0 =
insert_header(&backend, 3, block2, Some(changes2_2_0.clone()), Default::default());
let block2_2_1 =
insert_header(&backend, 4, block2_2_0, Some(changes2_2_1.clone()), Default::default());
// finalize block1
backend.changes_tries_storage.meta.write().finalized_number = 1;
@@ -680,7 +744,12 @@ mod tests {
if number == 0 {
Default::default()
} else {
backend.blockchain().header(BlockId::Number(number - 1)).unwrap().unwrap().hash()
backend
.blockchain()
.header(BlockId::Number(number - 1))
.unwrap()
.unwrap()
.hash()
}
};
@@ -698,12 +767,14 @@ mod tests {
let trie_root = backend
.blockchain()
.header(BlockId::Number(number))
.unwrap().unwrap()
.unwrap()
.unwrap()
.digest()
.log(DigestItem::as_changes_trie_root)
.cloned();
match trie_root {
Some(trie_root) => backend.changes_tries_storage.get(&trie_root, EMPTY_PREFIX).unwrap().is_none(),
Some(trie_root) =>
backend.changes_tries_storage.get(&trie_root, EMPTY_PREFIX).unwrap().is_none(),
None => true,
}
};
@@ -711,14 +782,10 @@ mod tests {
let finalize_block = |number| {
let header = backend.blockchain().header(BlockId::Number(number)).unwrap().unwrap();
let mut tx = Transaction::new();
let cache_ops = backend.changes_tries_storage.finalize(
&mut tx,
*header.parent_hash(),
header.hash(),
number,
None,
None,
).unwrap();
let cache_ops = backend
.changes_tries_storage
.finalize(&mut tx, *header.parent_hash(), header.hash(), number, None, None)
.unwrap();
backend.storage.db.commit(tx).unwrap();
backend.changes_tries_storage.post_commit(Some(cache_ops));
};
@@ -737,11 +804,23 @@ mod tests {
(0..6).for_each(|number| insert_regular_header(false, number));
insert_header_with_configuration_change(&backend, 6, parent_hash(6), None, config_at_6);
(7..17).for_each(|number| insert_regular_header(true, number));
insert_header_with_configuration_change(&backend, 17, parent_hash(17), changes(17), config_at_17);
insert_header_with_configuration_change(
&backend,
17,
parent_hash(17),
changes(17),
config_at_17,
);
(18..21).for_each(|number| insert_regular_header(false, number));
insert_header_with_configuration_change(&backend, 21, parent_hash(21), None, config_at_21);
(22..32).for_each(|number| insert_regular_header(true, number));
insert_header_with_configuration_change(&backend, 32, parent_hash(32), changes(32), config_at_32);
insert_header_with_configuration_change(
&backend,
32,
parent_hash(32),
changes(32),
config_at_32,
);
(33..50).for_each(|number| insert_regular_header(true, number));
// when only genesis is finalized, nothing is pruned
@@ -826,29 +905,24 @@ mod tests {
let backend = Backend::<Block>::new_test(1000, 100);
// configurations at blocks
let config_at_1 = Some(ChangesTrieConfiguration {
digest_interval: 4,
digest_levels: 2,
});
let config_at_3 = Some(ChangesTrieConfiguration {
digest_interval: 8,
digest_levels: 1,
});
let config_at_1 = Some(ChangesTrieConfiguration { digest_interval: 4, digest_levels: 2 });
let config_at_3 = Some(ChangesTrieConfiguration { digest_interval: 8, digest_levels: 1 });
let config_at_5 = None;
let config_at_7 = Some(ChangesTrieConfiguration {
digest_interval: 8,
digest_levels: 1,
});
let config_at_7 = Some(ChangesTrieConfiguration { digest_interval: 8, digest_levels: 1 });
// insert some blocks
let block0 = insert_header(&backend, 0, Default::default(), None, Default::default());
let block1 = insert_header_with_configuration_change(&backend, 1, block0, None, config_at_1.clone());
let block1 =
insert_header_with_configuration_change(&backend, 1, block0, None, config_at_1.clone());
let block2 = insert_header(&backend, 2, block1, None, Default::default());
let block3 = insert_header_with_configuration_change(&backend, 3, block2, None, config_at_3.clone());
let block3 =
insert_header_with_configuration_change(&backend, 3, block2, None, config_at_3.clone());
let block4 = insert_header(&backend, 4, block3, None, Default::default());
let block5 = insert_header_with_configuration_change(&backend, 5, block4, None, config_at_5.clone());
let block5 =
insert_header_with_configuration_change(&backend, 5, block4, None, config_at_5.clone());
let block6 = insert_header(&backend, 6, block5, None, Default::default());
let block7 = insert_header_with_configuration_change(&backend, 7, block6, None, config_at_7.clone());
let block7 =
insert_header_with_configuration_change(&backend, 7, block6, None, config_at_7.clone());
// test configuration cache
let storage = &backend.changes_tries_storage;
@@ -887,17 +961,48 @@ mod tests {
let mut backend = Backend::<Block>::new_test(10, 10);
backend.changes_tries_storage.min_blocks_to_keep = Some(8);
let configs = (0..=7).map(|i| Some(ChangesTrieConfiguration::new(2, i))).collect::<Vec<_>>();
let configs =
(0..=7).map(|i| Some(ChangesTrieConfiguration::new(2, i))).collect::<Vec<_>>();
// insert unfinalized headers
let block0 = insert_header_with_configuration_change(&backend, 0, Default::default(), None, configs[0].clone());
let block1 = insert_header_with_configuration_change(&backend, 1, block0, changes(1), configs[1].clone());
let block2 = insert_header_with_configuration_change(&backend, 2, block1, changes(2), configs[2].clone());
let block0 = insert_header_with_configuration_change(
&backend,
0,
Default::default(),
None,
configs[0].clone(),
);
let block1 = insert_header_with_configuration_change(
&backend,
1,
block0,
changes(1),
configs[1].clone(),
);
let block2 = insert_header_with_configuration_change(
&backend,
2,
block1,
changes(2),
configs[2].clone(),
);
let side_config2_1 = Some(ChangesTrieConfiguration::new(3, 2));
let side_config2_2 = Some(ChangesTrieConfiguration::new(3, 3));
let block2_1 = insert_header_with_configuration_change(&backend, 2, block1, changes(8), side_config2_1.clone());
let _ = insert_header_with_configuration_change(&backend, 3, block2_1, changes(9), side_config2_2.clone());
let block2_1 = insert_header_with_configuration_change(
&backend,
2,
block1,
changes(8),
side_config2_1.clone(),
);
let _ = insert_header_with_configuration_change(
&backend,
3,
block2_1,
changes(9),
side_config2_2.clone(),
);
// insert finalized header => 4 headers are finalized at once
let header3 = Header {
@@ -905,9 +1010,9 @@ mod tests {
parent_hash: block2,
state_root: Default::default(),
digest: Digest {
logs: vec![
DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(configs[3].clone())),
],
logs: vec![DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(
configs[3].clone(),
))],
},
extrinsics_root: Default::default(),
};
@@ -920,9 +1025,27 @@ mod tests {
backend.commit_operation(op).unwrap();
// insert more unfinalized headers
let block4 = insert_header_with_configuration_change(&backend, 4, block3, changes(4), configs[4].clone());
let block5 = insert_header_with_configuration_change(&backend, 5, block4, changes(5), configs[5].clone());
let block6 = insert_header_with_configuration_change(&backend, 6, block5, changes(6), configs[6].clone());
let block4 = insert_header_with_configuration_change(
&backend,
4,
block3,
changes(4),
configs[4].clone(),
);
let block5 = insert_header_with_configuration_change(
&backend,
5,
block4,
changes(5),
configs[5].clone(),
);
let block6 = insert_header_with_configuration_change(
&backend,
6,
block5,
changes(6),
configs[6].clone(),
);
// insert finalized header => 4 headers are finalized at once
let header7 = Header {
@@ -930,9 +1053,9 @@ mod tests {
parent_hash: block6,
state_root: Default::default(),
digest: Digest {
logs: vec![
DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(configs[7].clone())),
],
logs: vec![DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(
configs[7].clone(),
))],
},
extrinsics_root: Default::default(),
};
@@ -950,23 +1073,33 @@ mod tests {
let backend = Backend::<Block>::new_test(10, 10);
let config0 = Some(ChangesTrieConfiguration::new(2, 5));
let block0 = insert_header_with_configuration_change(&backend, 0, Default::default(), None, config0);
let block0 =
insert_header_with_configuration_change(&backend, 0, Default::default(), None, config0);
let config1 = Some(ChangesTrieConfiguration::new(2, 6));
let block1 = insert_header_with_configuration_change(&backend, 1, block0, changes(0), config1);
let block1 =
insert_header_with_configuration_change(&backend, 1, block0, changes(0), config1);
let just1 = Some((*b"TEST", vec![42]));
backend.finalize_block(BlockId::Number(1), just1).unwrap();
let config2 = Some(ChangesTrieConfiguration::new(2, 7));
let block2 = insert_header_with_configuration_change(&backend, 2, block1, changes(1), config2);
let block2 =
insert_header_with_configuration_change(&backend, 2, block1, changes(1), config2);
let config2_1 = Some(ChangesTrieConfiguration::new(2, 8));
let _ = insert_header_with_configuration_change(&backend, 3, block2, changes(10), config2_1);
let _ =
insert_header_with_configuration_change(&backend, 3, block2, changes(10), config2_1);
let config2_2 = Some(ChangesTrieConfiguration::new(2, 9));
let block2_2 = insert_header_with_configuration_change(&backend, 3, block2, changes(20), config2_2);
let block2_2 =
insert_header_with_configuration_change(&backend, 3, block2, changes(20), config2_2);
let config2_3 = Some(ChangesTrieConfiguration::new(2, 10));
let _ = insert_header_with_configuration_change(&backend, 4, block2_2, changes(30), config2_3);
let _ =
insert_header_with_configuration_change(&backend, 4, block2_2, changes(30), config2_3);
// before truncate there are 2 unfinalized forks - block2_1+block2_3
assert_eq!(
backend.changes_tries_storage.cache.0.write()
backend
.changes_tries_storage
.cache
.0
.write()
.get_cache(well_known_cache_keys::CHANGES_TRIE_CONFIG)
.unwrap()
.unfinalized()
@@ -979,7 +1112,11 @@ mod tests {
// after truncating block2_3 - there are 2 unfinalized forks - block2_1+block2_2
backend.revert(1, false).unwrap();
assert_eq!(
backend.changes_tries_storage.cache.0.write()
backend
.changes_tries_storage
.cache
.0
.write()
.get_cache(well_known_cache_keys::CHANGES_TRIE_CONFIG)
.unwrap()
.unfinalized()
@@ -993,7 +1130,11 @@ mod tests {
// the 1st one points to the block #3 because it isn't truncated
backend.revert(1, false).unwrap();
assert_eq!(
backend.changes_tries_storage.cache.0.write()
backend
.changes_tries_storage
.cache
.0
.write()
.get_cache(well_known_cache_keys::CHANGES_TRIE_CONFIG)
.unwrap()
.unfinalized()
@@ -1005,15 +1146,17 @@ mod tests {
// after truncating block2 - there are no unfinalized forks
backend.revert(1, false).unwrap();
assert!(
backend.changes_tries_storage.cache.0.write()
.get_cache(well_known_cache_keys::CHANGES_TRIE_CONFIG)
.unwrap()
.unfinalized()
.iter()
.map(|fork| fork.head().valid_from.number)
.collect::<Vec<_>>()
.is_empty(),
);
assert!(backend
.changes_tries_storage
.cache
.0
.write()
.get_cache(well_known_cache_keys::CHANGES_TRIE_CONFIG)
.unwrap()
.unfinalized()
.iter()
.map(|fork| fork.head().valid_from.number)
.collect::<Vec<_>>()
.is_empty(),);
}
}
+11 -9
View File
@@ -18,17 +18,22 @@
//! Functionality for reading and storing children hashes from db.
use codec::{Encode, Decode};
use sp_blockchain;
use std::hash::Hash;
use sp_database::{Database, Transaction};
use crate::DbHash;
use codec::{Decode, Encode};
use sp_blockchain;
use sp_database::{Database, Transaction};
use std::hash::Hash;
/// Returns the hashes of the children blocks of the block with `parent_hash`.
pub fn read_children<
K: Eq + Hash + Clone + Encode + Decode,
V: Eq + Hash + Clone + Encode + Decode,
>(db: &dyn Database<DbHash>, column: u32, prefix: &[u8], parent_hash: K) -> sp_blockchain::Result<Vec<V>> {
>(
db: &dyn Database<DbHash>,
column: u32,
prefix: &[u8],
parent_hash: K,
) -> sp_blockchain::Result<Vec<V>> {
let mut buf = prefix.to_vec();
parent_hash.using_encoded(|s| buf.extend(s));
@@ -65,9 +70,7 @@ pub fn write_children<
}
/// Prepare transaction to remove the children of `parent_hash`.
pub fn remove_children<
K: Eq + Hash + Clone + Encode + Decode,
>(
pub fn remove_children<K: Eq + Hash + Clone + Encode + Decode>(
tx: &mut Transaction<DbHash>,
column: u32,
prefix: &[u8],
@@ -78,7 +81,6 @@ pub fn remove_children<
tx.remove(column, &key);
}
#[cfg(test)]
mod tests {
use super::*;
File diff suppressed because it is too large Load Diff
+294 -203
View File
@@ -18,31 +18,31 @@
//! RocksDB-based light client blockchain storage.
use std::{sync::Arc, collections::HashMap};
use std::convert::TryInto;
use parking_lot::RwLock;
use std::{collections::HashMap, convert::TryInto, sync::Arc};
use crate::{
cache::{ComplexBlockId, DbCache, DbCacheSync, EntryType as CacheEntryType},
utils::{self, block_id_to_lookup_key, meta_keys, read_db, read_meta, DatabaseType, Meta},
DatabaseSettings, DbHash, FrozenForDuration,
};
use codec::{Decode, Encode};
use log::{debug, trace, warn};
use sc_client_api::{
cht, backend::{AuxStore, NewBlockState, ProvideChtRoots}, UsageInfo,
blockchain::{
BlockStatus, Cache as BlockchainCache, Info as BlockchainInfo,
},
Storage,
backend::{AuxStore, NewBlockState, ProvideChtRoots},
blockchain::{BlockStatus, Cache as BlockchainCache, Info as BlockchainInfo},
cht, Storage, UsageInfo,
};
use sp_blockchain::{
CachedHeaderMetadata, HeaderMetadata, HeaderMetadataCache,
Error as ClientError, Result as ClientResult,
HeaderBackend as BlockchainHeaderBackend,
well_known_cache_keys,
well_known_cache_keys, CachedHeaderMetadata, Error as ClientError,
HeaderBackend as BlockchainHeaderBackend, HeaderMetadata, HeaderMetadataCache,
Result as ClientResult,
};
use sp_database::{Database, Transaction};
use codec::{Decode, Encode};
use sp_runtime::generic::{DigestItem, BlockId};
use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, One, NumberFor, HashFor};
use crate::cache::{DbCacheSync, DbCache, ComplexBlockId, EntryType as CacheEntryType};
use crate::utils::{self, meta_keys, DatabaseType, Meta, read_db, block_id_to_lookup_key, read_meta};
use crate::{DatabaseSettings, FrozenForDuration, DbHash};
use log::{trace, warn, debug};
use sp_runtime::{
generic::{BlockId, DigestItem},
traits::{Block as BlockT, HashFor, Header as HeaderT, NumberFor, One, Zero},
};
pub(crate) mod columns {
pub const META: u32 = crate::utils::COLUMN_META;
@@ -139,8 +139,8 @@ impl<Block: BlockT> LightStorage<Block> {
}
impl<Block> BlockchainHeaderBackend<Block> for LightStorage<Block>
where
Block: BlockT,
where
Block: BlockT,
{
fn header(&self, id: BlockId<Block>) -> ClientResult<Option<Block::Header>> {
utils::read_header(&*self.db, columns::KEY_LOOKUP, columns::HEADER, id)
@@ -165,12 +165,8 @@ impl<Block> BlockchainHeaderBackend<Block> for LightStorage<Block>
fn status(&self, id: BlockId<Block>) -> ClientResult<BlockStatus> {
let exists = match id {
BlockId::Hash(_) => read_db(
&*self.db,
columns::KEY_LOOKUP,
columns::HEADER,
id
)?.is_some(),
BlockId::Hash(_) =>
read_db(&*self.db, columns::KEY_LOOKUP, columns::HEADER, id)?.is_some(),
BlockId::Number(n) => n <= self.meta.read().best_number,
};
match exists {
@@ -180,7 +176,9 @@ impl<Block> BlockchainHeaderBackend<Block> for LightStorage<Block>
}
fn number(&self, hash: Block::Hash) -> ClientResult<Option<NumberFor<Block>>> {
if let Some(lookup_key) = block_id_to_lookup_key::<Block>(&*self.db, columns::KEY_LOOKUP, BlockId::Hash(hash))? {
if let Some(lookup_key) =
block_id_to_lookup_key::<Block>(&*self.db, columns::KEY_LOOKUP, BlockId::Hash(hash))?
{
let number = utils::lookup_key_to_number(&lookup_key)?;
Ok(Some(number))
} else {
@@ -196,17 +194,25 @@ impl<Block> BlockchainHeaderBackend<Block> for LightStorage<Block>
impl<Block: BlockT> HeaderMetadata<Block> for LightStorage<Block> {
type Error = ClientError;
fn header_metadata(&self, hash: Block::Hash) -> Result<CachedHeaderMetadata<Block>, Self::Error> {
self.header_metadata_cache.header_metadata(hash).map_or_else(|| {
self.header(BlockId::hash(hash))?.map(|header| {
let header_metadata = CachedHeaderMetadata::from(&header);
self.header_metadata_cache.insert_header_metadata(
header_metadata.hash,
header_metadata.clone(),
);
header_metadata
}).ok_or_else(|| ClientError::UnknownBlock(format!("header not found in db: {}", hash)))
}, Ok)
fn header_metadata(
&self,
hash: Block::Hash,
) -> Result<CachedHeaderMetadata<Block>, Self::Error> {
self.header_metadata_cache.header_metadata(hash).map_or_else(
|| {
self.header(BlockId::hash(hash))?
.map(|header| {
let header_metadata = CachedHeaderMetadata::from(&header);
self.header_metadata_cache
.insert_header_metadata(header_metadata.hash, header_metadata.clone());
header_metadata
})
.ok_or_else(|| {
ClientError::UnknownBlock(format!("header not found in db: {}", hash))
})
},
Ok,
)
}
fn insert_header_metadata(&self, hash: Block::Hash, metadata: CachedHeaderMetadata<Block>) {
@@ -221,10 +227,9 @@ impl<Block: BlockT> HeaderMetadata<Block> for LightStorage<Block> {
impl<Block: BlockT> LightStorage<Block> {
// Get block changes trie root, if available.
fn changes_trie_root(&self, block: BlockId<Block>) -> ClientResult<Option<Block::Hash>> {
self.header(block)
.map(|header| header.and_then(|header|
header.digest().log(DigestItem::as_changes_trie_root)
.cloned()))
self.header(block).map(|header| {
header.and_then(|header| header.digest().log(DigestItem::as_changes_trie_root).cloned())
})
}
/// Handle setting head within a transaction. `route_to` should be the last
@@ -251,14 +256,16 @@ impl<Block: BlockT> LightStorage<Block> {
for retracted in tree_route.retracted() {
if retracted.hash == meta.finalized_hash {
// TODO: can we recover here?
warn!("Safety failure: reverting finalized block {:?}",
(&retracted.number, &retracted.hash));
warn!(
"Safety failure: reverting finalized block {:?}",
(&retracted.number, &retracted.hash)
);
}
utils::remove_number_to_key_mapping(
transaction,
columns::KEY_LOOKUP,
retracted.number
retracted.number,
)?;
}
@@ -267,7 +274,7 @@ impl<Block: BlockT> LightStorage<Block> {
transaction,
columns::KEY_LOOKUP,
enacted.number,
enacted.hash
enacted.hash,
)?;
}
}
@@ -292,10 +299,11 @@ impl<Block: BlockT> LightStorage<Block> {
) -> ClientResult<()> {
let meta = self.meta.read();
if &meta.finalized_hash != header.parent_hash() {
return Err(::sp_blockchain::Error::NonSequentialFinalization(
format!("Last finalized {:?} not parent of {:?}",
meta.finalized_hash, hash),
).into())
return Err(::sp_blockchain::Error::NonSequentialFinalization(format!(
"Last finalized {:?} not parent of {:?}",
meta.finalized_hash, hash
))
.into())
}
let lookup_key = utils::number_and_hash_to_lookup_key(header.number().clone(), hash)?;
@@ -313,12 +321,14 @@ impl<Block: BlockT> LightStorage<Block> {
});
let new_header_cht_root = cht::compute_root::<Block::Header, HashFor<Block>, _>(
cht::size(), new_cht_number, cht_range.map(|num| self.hash(num))
cht::size(),
new_cht_number,
cht_range.map(|num| self.hash(num)),
)?;
transaction.set(
columns::CHT,
&cht_key(HEADER_CHT_PREFIX, new_cht_start)?,
new_header_cht_root.as_ref()
new_header_cht_root.as_ref(),
);
// if the header includes changes trie root, let's build a changes tries roots CHT
@@ -329,14 +339,16 @@ impl<Block: BlockT> LightStorage<Block> {
current_num = current_num + One::one();
Some(old_current_num)
});
let new_changes_trie_cht_root = cht::compute_root::<Block::Header, HashFor<Block>, _>(
cht::size(), new_cht_number, cht_range
.map(|num| self.changes_trie_root(BlockId::Number(num)))
)?;
let new_changes_trie_cht_root =
cht::compute_root::<Block::Header, HashFor<Block>, _>(
cht::size(),
new_cht_number,
cht_range.map(|num| self.changes_trie_root(BlockId::Number(num))),
)?;
transaction.set(
columns::CHT,
&cht_key(CHANGES_TRIE_CHT_PREFIX, new_cht_start)?,
new_changes_trie_cht_root.as_ref()
new_changes_trie_cht_root.as_ref(),
);
}
@@ -354,7 +366,7 @@ impl<Block: BlockT> LightStorage<Block> {
transaction,
columns::KEY_LOOKUP,
prune_block,
hash
hash,
)?;
transaction.remove(columns::HEADER, &lookup_key);
}
@@ -370,7 +382,7 @@ impl<Block: BlockT> LightStorage<Block> {
&self,
cht_type: u8,
cht_size: NumberFor<Block>,
block: NumberFor<Block>
block: NumberFor<Block>,
) -> ClientResult<Option<Block::Hash>> {
let no_cht_for_block = || ClientError::Backend(format!("Missing CHT for block {}", block));
@@ -383,7 +395,8 @@ impl<Block: BlockT> LightStorage<Block> {
}
let cht_start = cht::start_number(cht_size, cht_number);
self.db.get(columns::CHT, &cht_key(cht_type, cht_start)?)
self.db
.get(columns::CHT, &cht_key(cht_type, cht_start)?)
.ok_or_else(no_cht_for_block)
.and_then(|hash| Block::Hash::decode(&mut &*hash).map_err(|_| no_cht_for_block()))
.map(Some)
@@ -391,15 +404,20 @@ impl<Block: BlockT> LightStorage<Block> {
}
impl<Block> AuxStore for LightStorage<Block>
where Block: BlockT,
where
Block: BlockT,
{
fn insert_aux<
'a,
'b: 'a,
'c: 'a,
I: IntoIterator<Item=&'a(&'c [u8], &'c [u8])>,
D: IntoIterator<Item=&'a &'b [u8]>,
>(&self, insert: I, delete: D) -> ClientResult<()> {
I: IntoIterator<Item = &'a (&'c [u8], &'c [u8])>,
D: IntoIterator<Item = &'a &'b [u8]>,
>(
&self,
insert: I,
delete: D,
) -> ClientResult<()> {
let mut transaction = Transaction::new();
for (k, v) in insert {
transaction.set(columns::AUX, k, v);
@@ -418,7 +436,8 @@ impl<Block> AuxStore for LightStorage<Block>
}
impl<Block> Storage<Block> for LightStorage<Block>
where Block: BlockT,
where
Block: BlockT,
{
fn import_header(
&self,
@@ -447,19 +466,12 @@ impl<Block> Storage<Block> for LightStorage<Block>
self.set_head_with_transaction(&mut transaction, parent_hash, (number, hash))?;
}
utils::insert_hash_to_key_mapping(
&mut transaction,
columns::KEY_LOOKUP,
number,
hash,
)?;
utils::insert_hash_to_key_mapping(&mut transaction, columns::KEY_LOOKUP, number, hash)?;
transaction.set_from_vec(columns::HEADER, &lookup_key, header.encode());
let header_metadata = CachedHeaderMetadata::from(&header);
self.header_metadata_cache.insert_header_metadata(
header.hash().clone(),
header_metadata,
);
self.header_metadata_cache
.insert_header_metadata(header.hash().clone(), header_metadata);
let is_genesis = number.is_zero();
if is_genesis {
@@ -474,25 +486,28 @@ impl<Block> Storage<Block> for LightStorage<Block>
};
if finalized {
self.note_finalized(
&mut transaction,
&header,
hash,
)?;
self.note_finalized(&mut transaction, &header, hash)?;
}
// update changes trie configuration cache
if !cache_at.contains_key(&well_known_cache_keys::CHANGES_TRIE_CONFIG) {
if let Some(new_configuration) = crate::changes_tries_storage::extract_new_configuration(&header) {
cache_at.insert(well_known_cache_keys::CHANGES_TRIE_CONFIG, new_configuration.encode());
if let Some(new_configuration) =
crate::changes_tries_storage::extract_new_configuration(&header)
{
cache_at
.insert(well_known_cache_keys::CHANGES_TRIE_CONFIG, new_configuration.encode());
}
}
{
let mut cache = self.cache.0.write();
let cache_ops = cache.transaction(&mut transaction)
let cache_ops = cache
.transaction(&mut transaction)
.on_block_insert(
ComplexBlockId::new(*header.parent_hash(), if number.is_zero() { Zero::zero() } else { number - One::one() }),
ComplexBlockId::new(
*header.parent_hash(),
if number.is_zero() { Zero::zero() } else { number - One::one() },
),
ComplexBlockId::new(hash, number),
cache_at,
if finalized { CacheEntryType::Final } else { CacheEntryType::NonFinal },
@@ -502,9 +517,10 @@ impl<Block> Storage<Block> for LightStorage<Block>
debug!("Light DB Commit {:?} ({})", hash, number);
self.db.commit(transaction)?;
cache.commit(cache_ops)
.expect("only fails if cache with given name isn't loaded yet;\
cache is already loaded because there are cache_ops; qed");
cache.commit(cache_ops).expect(
"only fails if cache with given name isn't loaded yet;\
cache is already loaded because there are cache_ops; qed",
);
}
self.update_meta(hash, number, leaf_state.is_best(), finalized);
@@ -518,7 +534,11 @@ impl<Block> Storage<Block> for LightStorage<Block>
let number = header.number();
let mut transaction = Transaction::new();
self.set_head_with_transaction(&mut transaction, hash.clone(), (number.clone(), hash.clone()))?;
self.set_head_with_transaction(
&mut transaction,
hash.clone(),
(number.clone(), hash.clone()),
)?;
self.db.commit(transaction)?;
self.update_meta(hash, header.number().clone(), true, false);
@@ -536,17 +556,22 @@ impl<Block> Storage<Block> for LightStorage<Block>
self.note_finalized(&mut transaction, &header, hash.clone())?;
{
let mut cache = self.cache.0.write();
let cache_ops = cache.transaction(&mut transaction)
let cache_ops = cache
.transaction(&mut transaction)
.on_block_finalize(
ComplexBlockId::new(*header.parent_hash(), if number.is_zero() { Zero::zero() } else { number - One::one() }),
ComplexBlockId::new(hash, number)
ComplexBlockId::new(
*header.parent_hash(),
if number.is_zero() { Zero::zero() } else { number - One::one() },
),
ComplexBlockId::new(hash, number),
)?
.into_ops();
self.db.commit(transaction)?;
cache.commit(cache_ops)
.expect("only fails if cache with given name isn't loaded yet;\
cache is already loaded because there are cache_ops; qed");
cache.commit(cache_ops).expect(
"only fails if cache with given name isn't loaded yet;\
cache is already loaded because there are cache_ops; qed",
);
}
self.update_meta(hash, header.number().clone(), false, true);
@@ -566,7 +591,7 @@ impl<Block> Storage<Block> for LightStorage<Block>
#[cfg(not(target_os = "unknown"))]
fn usage_info(&self) -> Option<UsageInfo> {
use sc_client_api::{MemoryInfo, IoInfo, MemorySize};
use sc_client_api::{IoInfo, MemoryInfo, MemorySize};
// TODO: reimplement IO stats
let database_cache = MemorySize::from_bytes(0);
@@ -591,7 +616,7 @@ impl<Block> Storage<Block> for LightStorage<Block>
state_reads_cache: 0,
state_writes_cache: 0,
state_writes_nodes: 0,
}
},
})
}
@@ -602,7 +627,8 @@ impl<Block> Storage<Block> for LightStorage<Block>
}
impl<Block> ProvideChtRoots<Block> for LightStorage<Block>
where Block: BlockT,
where
Block: BlockT,
{
fn header_cht_root(
&self,
@@ -630,12 +656,14 @@ fn cht_key<N: TryInto<u32>>(cht_type: u8, block: N) -> ClientResult<[u8; 5]> {
#[cfg(test)]
pub(crate) mod tests {
use sc_client_api::cht;
use sp_core::ChangesTrieConfiguration;
use sp_runtime::generic::{DigestItem, ChangesTrieSignal};
use sp_runtime::testing::{H256 as Hash, Header, Block as RawBlock, ExtrinsicWrapper};
use sp_blockchain::{lowest_common_ancestor, tree_route};
use super::*;
use sc_client_api::cht;
use sp_blockchain::{lowest_common_ancestor, tree_route};
use sp_core::ChangesTrieConfiguration;
use sp_runtime::{
generic::{ChangesTrieSignal, DigestItem},
testing::{Block as RawBlock, ExtrinsicWrapper, Header, H256 as Hash},
};
type Block = RawBlock<ExtrinsicWrapper<u32>>;
type AuthorityId = sp_core::ed25519::Public;
@@ -652,7 +680,10 @@ pub(crate) mod tests {
fn header_with_changes_trie(parent: &Hash, number: u64) -> Header {
let mut header = default_header(parent, number);
header.digest.logs.push(DigestItem::ChangesTrieRoot([(number % 256) as u8; 32].into()));
header
.digest
.logs
.push(DigestItem::ChangesTrieRoot([(number % 256) as u8; 32].into()));
header
}
@@ -698,7 +729,8 @@ pub(crate) mod tests {
#[test]
fn returns_known_header() {
let db = LightStorage::new_test();
let known_hash = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let known_hash =
insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let header_by_hash = db.header(BlockId::Hash(known_hash)).unwrap().unwrap();
let header_by_number = db.header(BlockId::Number(0)).unwrap().unwrap();
assert_eq!(header_by_hash, header_by_number);
@@ -714,7 +746,8 @@ pub(crate) mod tests {
#[test]
fn returns_info() {
let db = LightStorage::new_test();
let genesis_hash = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let genesis_hash =
insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let info = db.info();
assert_eq!(info.best_hash, genesis_hash);
assert_eq!(info.best_number, 0);
@@ -729,17 +762,22 @@ pub(crate) mod tests {
#[test]
fn returns_block_status() {
let db = LightStorage::new_test();
let genesis_hash = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let genesis_hash =
insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
assert_eq!(db.status(BlockId::Hash(genesis_hash)).unwrap(), BlockStatus::InChain);
assert_eq!(db.status(BlockId::Number(0)).unwrap(), BlockStatus::InChain);
assert_eq!(db.status(BlockId::Hash(Hash::from_low_u64_be(1))).unwrap(), BlockStatus::Unknown);
assert_eq!(
db.status(BlockId::Hash(Hash::from_low_u64_be(1))).unwrap(),
BlockStatus::Unknown
);
assert_eq!(db.status(BlockId::Number(1)).unwrap(), BlockStatus::Unknown);
}
#[test]
fn returns_block_hash() {
let db = LightStorage::new_test();
let genesis_hash = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let genesis_hash =
insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
assert_eq!(db.hash(0).unwrap(), Some(genesis_hash));
assert_eq!(db.hash(1).unwrap(), None);
}
@@ -749,7 +787,8 @@ pub(crate) mod tests {
let raw_db = Arc::new(sp_database::MemDb::default());
let db = LightStorage::from_kvdb(raw_db.clone()).unwrap();
let genesis_hash = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let genesis_hash =
insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
assert_eq!(raw_db.count(columns::HEADER), 1);
assert_eq!(raw_db.count(columns::KEY_LOOKUP), 2);
@@ -760,43 +799,41 @@ pub(crate) mod tests {
#[test]
fn finalized_ancient_headers_are_replaced_with_cht() {
fn insert_headers<F: Fn(&Hash, u64) -> Header>(header_producer: F) ->
(Arc<sp_database::MemDb>, LightStorage<Block>)
{
fn insert_headers<F: Fn(&Hash, u64) -> Header>(
header_producer: F,
) -> (Arc<sp_database::MemDb>, LightStorage<Block>) {
let raw_db = Arc::new(sp_database::MemDb::default());
let db = LightStorage::from_kvdb(raw_db.clone()).unwrap();
let cht_size: u64 = cht::size();
let ucht_size: usize = cht_size as _;
// insert genesis block header (never pruned)
let mut prev_hash = insert_final_block(&db, HashMap::new(), || header_producer(&Default::default(), 0));
let mut prev_hash =
insert_final_block(&db, HashMap::new(), || header_producer(&Default::default(), 0));
// insert SIZE blocks && ensure that nothing is pruned
for number in 0..cht::size() {
prev_hash = insert_block(&db, HashMap::new(), || header_producer(&prev_hash, 1 + number));
prev_hash =
insert_block(&db, HashMap::new(), || header_producer(&prev_hash, 1 + number));
}
assert_eq!(raw_db.count(columns::HEADER), 1 + ucht_size);
assert_eq!(raw_db.count(columns::CHT), 0);
// insert next SIZE blocks && ensure that nothing is pruned
for number in 0..(cht_size as _) {
prev_hash = insert_block(
&db,
HashMap::new(),
|| header_producer(&prev_hash, 1 + cht_size + number),
);
prev_hash = insert_block(&db, HashMap::new(), || {
header_producer(&prev_hash, 1 + cht_size + number)
});
}
assert_eq!(raw_db.count(columns::HEADER), 1 + ucht_size + ucht_size);
assert_eq!(raw_db.count(columns::CHT), 0);
// insert block #{2 * cht::size() + 1} && check that new CHT is created + headers of this CHT are pruned
// nothing is yet finalized, so nothing is pruned.
prev_hash = insert_block(
&db,
HashMap::new(),
|| header_producer(&prev_hash, 1 + cht_size + cht_size),
);
prev_hash = insert_block(&db, HashMap::new(), || {
header_producer(&prev_hash, 1 + cht_size + cht_size)
});
assert_eq!(raw_db.count(columns::HEADER), 2 + ucht_size + ucht_size);
assert_eq!(raw_db.count(columns::CHT), 0);
@@ -839,7 +876,10 @@ pub(crate) mod tests {
#[test]
fn get_cht_fails_for_non_existent_cht() {
let cht_size: u64 = cht::size();
assert!(LightStorage::<Block>::new_test().header_cht_root(cht_size, cht_size / 2).unwrap().is_none());
assert!(LightStorage::<Block>::new_test()
.header_cht_root(cht_size, cht_size / 2)
.unwrap()
.is_none());
}
#[test]
@@ -847,26 +887,41 @@ pub(crate) mod tests {
let db = LightStorage::new_test();
// insert 1 + SIZE + SIZE + 1 blocks so that CHT#0 is created
let mut prev_hash = insert_final_block(&db, HashMap::new(), || header_with_changes_trie(&Default::default(), 0));
let mut prev_hash = insert_final_block(&db, HashMap::new(), || {
header_with_changes_trie(&Default::default(), 0)
});
let cht_size: u64 = cht::size();
let ucht_size: usize = cht_size as _;
for i in 1..1 + ucht_size + ucht_size + 1 {
prev_hash = insert_block(&db, HashMap::new(), || header_with_changes_trie(&prev_hash, i as u64));
prev_hash = insert_block(&db, HashMap::new(), || {
header_with_changes_trie(&prev_hash, i as u64)
});
db.finalize_header(BlockId::Hash(prev_hash)).unwrap();
}
let cht_root_1 = db.header_cht_root(cht_size, cht::start_number(cht_size, 0)).unwrap().unwrap();
let cht_root_2 = db.header_cht_root(cht_size, cht::start_number(cht_size, 0) + cht_size / 2).unwrap().unwrap();
let cht_root_3 = db.header_cht_root(cht_size, cht::end_number(cht_size, 0)).unwrap().unwrap();
let cht_root_1 =
db.header_cht_root(cht_size, cht::start_number(cht_size, 0)).unwrap().unwrap();
let cht_root_2 = db
.header_cht_root(cht_size, cht::start_number(cht_size, 0) + cht_size / 2)
.unwrap()
.unwrap();
let cht_root_3 =
db.header_cht_root(cht_size, cht::end_number(cht_size, 0)).unwrap().unwrap();
assert_eq!(cht_root_1, cht_root_2);
assert_eq!(cht_root_2, cht_root_3);
let cht_root_1 = db.changes_trie_cht_root(cht_size, cht::start_number(cht_size, 0)).unwrap().unwrap();
let cht_root_2 = db.changes_trie_cht_root(
cht_size,
cht::start_number(cht_size, 0) + cht_size / 2,
).unwrap().unwrap();
let cht_root_3 = db.changes_trie_cht_root(cht_size, cht::end_number(cht_size, 0)).unwrap().unwrap();
let cht_root_1 = db
.changes_trie_cht_root(cht_size, cht::start_number(cht_size, 0))
.unwrap()
.unwrap();
let cht_root_2 = db
.changes_trie_cht_root(cht_size, cht::start_number(cht_size, 0) + cht_size / 2)
.unwrap()
.unwrap();
let cht_root_3 = db
.changes_trie_cht_root(cht_size, cht::end_number(cht_size, 0))
.unwrap()
.unwrap();
assert_eq!(cht_root_1, cht_root_2);
assert_eq!(cht_root_2, cht_root_3);
}
@@ -882,15 +937,23 @@ pub(crate) mod tests {
let a3 = insert_block(&db, HashMap::new(), || default_header(&a2, 3));
// fork from genesis: 2 prong.
let b1 = insert_block(&db, HashMap::new(), || header_with_extrinsics_root(&block0, 1, Hash::from([1; 32])));
let b1 = insert_block(&db, HashMap::new(), || {
header_with_extrinsics_root(&block0, 1, Hash::from([1; 32]))
});
let b2 = insert_block(&db, HashMap::new(), || default_header(&b1, 2));
{
let tree_route = tree_route(&db, a3, b2).unwrap();
assert_eq!(tree_route.common_block().hash, block0);
assert_eq!(tree_route.retracted().iter().map(|r| r.hash).collect::<Vec<_>>(), vec![a3, a2, a1]);
assert_eq!(tree_route.enacted().iter().map(|r| r.hash).collect::<Vec<_>>(), vec![b1, b2]);
assert_eq!(
tree_route.retracted().iter().map(|r| r.hash).collect::<Vec<_>>(),
vec![a3, a2, a1]
);
assert_eq!(
tree_route.enacted().iter().map(|r| r.hash).collect::<Vec<_>>(),
vec![b1, b2]
);
}
{
@@ -898,14 +961,20 @@ pub(crate) mod tests {
assert_eq!(tree_route.common_block().hash, a1);
assert!(tree_route.retracted().is_empty());
assert_eq!(tree_route.enacted().iter().map(|r| r.hash).collect::<Vec<_>>(), vec![a2, a3]);
assert_eq!(
tree_route.enacted().iter().map(|r| r.hash).collect::<Vec<_>>(),
vec![a2, a3]
);
}
{
let tree_route = tree_route(&db, a3, a1).unwrap();
assert_eq!(tree_route.common_block().hash, a1);
assert_eq!(tree_route.retracted().iter().map(|r| r.hash).collect::<Vec<_>>(), vec![a3, a2]);
assert_eq!(
tree_route.retracted().iter().map(|r| r.hash).collect::<Vec<_>>(),
vec![a3, a2]
);
assert!(tree_route.enacted().is_empty());
}
@@ -929,7 +998,9 @@ pub(crate) mod tests {
let a3 = insert_block(&db, HashMap::new(), || default_header(&a2, 3));
// fork from genesis: 2 prong.
let b1 = insert_block(&db, HashMap::new(), || header_with_extrinsics_root(&block0, 1, Hash::from([1; 32])));
let b1 = insert_block(&db, HashMap::new(), || {
header_with_extrinsics_root(&block0, 1, Hash::from([1; 32]))
});
let b2 = insert_block(&db, HashMap::new(), || default_header(&b1, 2));
{
@@ -979,7 +1050,11 @@ pub(crate) mod tests {
fn authorities_are_cached() {
let db = LightStorage::new_test();
fn run_checks(db: &LightStorage<Block>, max: u64, checks: &[(u64, Option<Vec<AuthorityId>>)]) {
fn run_checks(
db: &LightStorage<Block>,
max: u64,
checks: &[(u64, Option<Vec<AuthorityId>>)],
) {
for (at, expected) in checks.iter().take_while(|(at, _)| *at <= max) {
let actual = authorities(db.cache(), BlockId::Number(*at));
assert_eq!(*expected, actual);
@@ -990,14 +1065,21 @@ pub(crate) mod tests {
HashMap::new()
}
fn make_authorities(authorities: Vec<AuthorityId>) -> HashMap<well_known_cache_keys::Id, Vec<u8>> {
fn make_authorities(
authorities: Vec<AuthorityId>,
) -> HashMap<well_known_cache_keys::Id, Vec<u8>> {
let mut map = HashMap::new();
map.insert(well_known_cache_keys::AUTHORITIES, authorities.encode());
map
}
fn authorities(cache: &dyn BlockchainCache<Block>, at: BlockId<Block>) -> Option<Vec<AuthorityId>> {
cache.get_at(&well_known_cache_keys::AUTHORITIES, &at).unwrap_or(None)
fn authorities(
cache: &dyn BlockchainCache<Block>,
at: BlockId<Block>,
) -> Option<Vec<AuthorityId>> {
cache
.get_at(&well_known_cache_keys::AUTHORITIES, &at)
.unwrap_or(None)
.and_then(|(_, _, val)| Decode::decode(&mut &val[..]).ok())
}
@@ -1021,17 +1103,27 @@ pub(crate) mod tests {
(6, Some(vec![auth1(), auth2()])),
];
let hash0 = insert_final_block(&db, same_authorities(), || default_header(&Default::default(), 0));
let hash0 = insert_final_block(&db, same_authorities(), || {
default_header(&Default::default(), 0)
});
run_checks(&db, 0, &checks);
let hash1 = insert_final_block(&db, same_authorities(), || default_header(&hash0, 1));
run_checks(&db, 1, &checks);
let hash2 = insert_final_block(&db, make_authorities(vec![auth1()]), || default_header(&hash1, 2));
let hash2 = insert_final_block(&db, make_authorities(vec![auth1()]), || {
default_header(&hash1, 2)
});
run_checks(&db, 2, &checks);
let hash3 = insert_final_block(&db, make_authorities(vec![auth1()]), || default_header(&hash2, 3));
let hash3 = insert_final_block(&db, make_authorities(vec![auth1()]), || {
default_header(&hash2, 3)
});
run_checks(&db, 3, &checks);
let hash4 = insert_final_block(&db, make_authorities(vec![auth1(), auth2()]), || default_header(&hash3, 4));
let hash4 = insert_final_block(&db, make_authorities(vec![auth1(), auth2()]), || {
default_header(&hash3, 4)
});
run_checks(&db, 4, &checks);
let hash5 = insert_final_block(&db, make_authorities(vec![auth1(), auth2()]), || default_header(&hash4, 5));
let hash5 = insert_final_block(&db, make_authorities(vec![auth1(), auth2()]), || {
default_header(&hash4, 5)
});
run_checks(&db, 5, &checks);
let hash6 = insert_final_block(&db, same_authorities(), || default_header(&hash5, 6));
run_checks(&db, 6, &checks);
@@ -1043,9 +1135,14 @@ pub(crate) mod tests {
// some older non-best blocks are inserted
// ... -> B2(1) -> B2_1(1) -> B2_2(2)
// => the cache ignores all writes before best finalized block
let hash2_1 = insert_non_best_block(&db, make_authorities(vec![auth1()]), || default_header(&hash2, 3));
let hash2_1 = insert_non_best_block(&db, make_authorities(vec![auth1()]), || {
default_header(&hash2, 3)
});
assert_eq!(None, authorities(db.cache(), BlockId::Hash(hash2_1)));
let hash2_2 = insert_non_best_block(&db, make_authorities(vec![auth1(), auth2()]), || default_header(&hash2_1, 4));
let hash2_2 =
insert_non_best_block(&db, make_authorities(vec![auth1(), auth2()]), || {
default_header(&hash2_1, 4)
});
assert_eq!(None, authorities(db.cache(), BlockId::Hash(hash2_2)));
}
@@ -1056,51 +1153,41 @@ pub(crate) mod tests {
// \> B6_1_1(5)
// \> B6_1_2(6) -> B6_1_3(7)
let hash7 = insert_block(&db, make_authorities(vec![auth3()]), || default_header(&hash6, 7));
assert_eq!(
authorities(db.cache(), BlockId::Hash(hash6)),
Some(vec![auth1(), auth2()]),
);
let hash7 =
insert_block(&db, make_authorities(vec![auth3()]), || default_header(&hash6, 7));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]),);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()]));
let hash8 = insert_block(&db, make_authorities(vec![auth3()]), || default_header(&hash7, 8));
assert_eq!(
authorities(db.cache(), BlockId::Hash(hash6)),
Some(vec![auth1(), auth2()]),
);
let hash8 =
insert_block(&db, make_authorities(vec![auth3()]), || default_header(&hash7, 8));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]),);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()]));
let hash6_1 = insert_block(&db, make_authorities(vec![auth4()]), || default_header(&hash6, 7));
assert_eq!(
authorities(db.cache(), BlockId::Hash(hash6)),
Some(vec![auth1(), auth2()]),
);
let hash6_1 =
insert_block(&db, make_authorities(vec![auth4()]), || default_header(&hash6, 7));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]),);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()]));
let hash6_1_1 = insert_non_best_block(&db, make_authorities(vec![auth5()]), || default_header(&hash6_1, 8));
assert_eq!(
authorities(db.cache(), BlockId::Hash(hash6)),
Some(vec![auth1(), auth2()]),
);
let hash6_1_1 = insert_non_best_block(&db, make_authorities(vec![auth5()]), || {
default_header(&hash6_1, 8)
});
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]),);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()]));
let hash6_1_2 = insert_non_best_block(&db, make_authorities(vec![auth6()]), || default_header(&hash6_1, 8));
assert_eq!(
authorities(db.cache(), BlockId::Hash(hash6)),
Some(vec![auth1(), auth2()]),
);
let hash6_1_2 = insert_non_best_block(&db, make_authorities(vec![auth6()]), || {
default_header(&hash6_1, 8)
});
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]),);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_2)), Some(vec![auth6()]));
let hash6_2 = insert_block(&db, make_authorities(vec![auth4()]), || default_header(&hash6_1, 8));
assert_eq!(
authorities(db.cache(), BlockId::Hash(hash6)),
Some(vec![auth1(), auth2()]),
);
let hash6_2 =
insert_block(&db, make_authorities(vec![auth4()]), || default_header(&hash6_1, 8));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]),);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()]));
@@ -1114,10 +1201,7 @@ pub(crate) mod tests {
{
// finalize block hash6_1
db.finalize_header(BlockId::Hash(hash6_1)).unwrap();
assert_eq!(
authorities(db.cache(), BlockId::Hash(hash6)),
Some(vec![auth1(), auth2()]),
);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]),);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), None);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), None);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()]));
@@ -1126,10 +1210,7 @@ pub(crate) mod tests {
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_2)), Some(vec![auth4()]));
// finalize block hash6_2
db.finalize_header(BlockId::Hash(hash6_2)).unwrap();
assert_eq!(
authorities(db.cache(), BlockId::Hash(hash6)),
Some(vec![auth1(), auth2()]),
);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]),);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), None);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), None);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()]));
@@ -1142,7 +1223,8 @@ pub(crate) mod tests {
#[test]
fn database_is_reopened() {
let db = LightStorage::new_test();
let hash0 = insert_final_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let hash0 =
insert_final_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
assert_eq!(db.info().best_hash, hash0);
assert_eq!(db.header(BlockId::Hash(hash0)).unwrap().unwrap().hash(), hash0);
@@ -1157,7 +1239,8 @@ pub(crate) mod tests {
let db = LightStorage::<Block>::new_test();
// insert aux1 + aux2 using direct store access
db.insert_aux(&[(&[1][..], &[101][..]), (&[2][..], &[102][..])], ::std::iter::empty()).unwrap();
db.insert_aux(&[(&[1][..], &[101][..]), (&[2][..], &[102][..])], ::std::iter::empty())
.unwrap();
// check aux values
assert_eq!(db.get_aux(&[1]).unwrap(), Some(vec![101]));
@@ -1165,10 +1248,13 @@ pub(crate) mod tests {
assert_eq!(db.get_aux(&[3]).unwrap(), None);
// delete aux1 + insert aux3 using import operation
db.import_header(default_header(&Default::default(), 0), HashMap::new(), NewBlockState::Best, vec![
(vec![3], Some(vec![103])),
(vec![1], None),
]).unwrap();
db.import_header(
default_header(&Default::default(), 0),
HashMap::new(),
NewBlockState::Best,
vec![(vec![3], Some(vec![103])), (vec![1], None)],
)
.unwrap();
// check aux values
assert_eq!(db.get_aux(&[1]).unwrap(), None);
@@ -1208,7 +1294,8 @@ pub(crate) mod tests {
};
// restart && check that after restart value is read from the cache
let db = LightStorage::<Block>::from_kvdb(storage as Arc<_>).expect("failed to create test-db");
let db =
LightStorage::<Block>::from_kvdb(storage as Arc<_>).expect("failed to create test-db");
assert_eq!(
db.cache().get_at(b"test", &BlockId::Number(0)).unwrap(),
Some(((0, genesis_hash.unwrap()), None, vec![42])),
@@ -1224,7 +1311,9 @@ pub(crate) mod tests {
// insert block#0 && block#1 (no value for cache is provided)
let hash0 = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
assert_eq!(
db.cache().get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(0)).unwrap()
db.cache()
.get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(0))
.unwrap()
.map(|(_, _, v)| ChangesTrieConfiguration::decode(&mut &v[..]).unwrap()),
None,
);
@@ -1232,13 +1321,15 @@ pub(crate) mod tests {
// insert configuration at block#1 (starts from block#2)
insert_block(&db, HashMap::new(), || {
let mut header = default_header(&hash0, 1);
header.digest_mut().push(
DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(new_config.clone()))
);
header.digest_mut().push(DigestItem::ChangesTrieSignal(
ChangesTrieSignal::NewConfiguration(new_config.clone()),
));
header
});
assert_eq!(
db.cache().get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(1)).unwrap()
db.cache()
.get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(1))
.unwrap()
.map(|(_, _, v)| Option::<ChangesTrieConfiguration>::decode(&mut &v[..]).unwrap()),
Some(new_config),
);
+4 -13
View File
@@ -21,8 +21,8 @@
use std::{collections::HashMap, sync::Arc};
use crate::{columns, Database, DbHash, Transaction};
use parking_lot::Mutex;
use log::error;
use parking_lot::Mutex;
/// Offchain local storage
#[derive(Clone)]
@@ -33,8 +33,7 @@ pub struct LocalStorage {
impl std::fmt::Debug for LocalStorage {
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
fmt.debug_struct("LocalStorage")
.finish()
fmt.debug_struct("LocalStorage").finish()
}
}
@@ -49,10 +48,7 @@ impl LocalStorage {
/// Create offchain local storage with given `KeyValueDB` backend.
pub fn new(db: Arc<dyn Database<DbHash>>) -> Self {
Self {
db,
locks: Default::default(),
}
Self { db, locks: Default::default() }
}
}
@@ -118,11 +114,7 @@ impl sp_core::offchain::OffchainStorage for LocalStorage {
/// Concatenate the prefix and key to create an offchain key in the db.
pub(crate) fn concatenate_prefix_and_key(prefix: &[u8], key: &[u8]) -> Vec<u8> {
prefix
.iter()
.chain(key.into_iter())
.cloned()
.collect()
prefix.iter().chain(key.into_iter()).cloned().collect()
}
#[cfg(test)]
@@ -155,5 +147,4 @@ mod tests {
assert_eq!(storage.get(prefix, key), Some(b"asd".to_vec()));
assert!(storage.locks.lock().is_empty(), "Locks map should be empty!");
}
}
+16 -16
View File
@@ -15,27 +15,29 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
columns,
utils::{DatabaseType, NUM_COLUMNS},
};
/// A `Database` adapter for parity-db.
use sp_database::{Database, Change, ColumnId, Transaction, error::DatabaseError};
use crate::utils::{DatabaseType, NUM_COLUMNS};
use crate::columns;
use sp_database::{error::DatabaseError, Change, ColumnId, Database, Transaction};
struct DbAdapter(parity_db::Db);
fn handle_err<T>(result: parity_db::Result<T>) -> T {
match result {
Ok(r) => r,
Err(e) => {
Err(e) => {
panic!("Critical database error: {:?}", e);
}
},
}
}
/// Wrap parity-db database into a trait object that implements `sp_database::Database`
pub fn open<H: Clone + AsRef<[u8]>>(path: &std::path::Path, db_type: DatabaseType)
-> parity_db::Result<std::sync::Arc<dyn Database<H>>>
{
pub fn open<H: Clone + AsRef<[u8]>>(
path: &std::path::Path,
db_type: DatabaseType,
) -> parity_db::Result<std::sync::Arc<dyn Database<H>>> {
let mut config = parity_db::Options::with_columns(path, NUM_COLUMNS as u8);
config.sync = true; // Flush each commit
if db_type == DatabaseType::Full {
@@ -50,13 +52,11 @@ pub fn open<H: Clone + AsRef<[u8]>>(path: &std::path::Path, db_type: DatabaseTyp
impl<H: Clone + AsRef<[u8]>> Database<H> for DbAdapter {
fn commit(&self, transaction: Transaction<H>) -> Result<(), DatabaseError> {
handle_err(self.0.commit(transaction.0.into_iter().map(|change|
match change {
Change::Set(col, key, value) => (col as u8, key, Some(value)),
Change::Remove(col, key) => (col as u8, key, None),
_ => unimplemented!(),
}))
);
handle_err(self.0.commit(transaction.0.into_iter().map(|change| match change {
Change::Set(col, key, value) => (col as u8, key, Some(value)),
Change::Remove(col, key) => (col as u8, key, None),
_ => unimplemented!(),
})));
Ok(())
}
+8 -3
View File
@@ -65,7 +65,10 @@ impl StateUsageStats {
/// Tally one key read.
pub fn tally_key_read(&self, key: &[u8], val: Option<&Vec<u8>>, cache: bool) {
self.tally_read(key.len() as u64 + val.as_ref().map(|x| x.len() as u64).unwrap_or(0), cache);
self.tally_read(
key.len() as u64 + val.as_ref().map(|x| x.len() as u64).unwrap_or(0),
cache,
);
}
/// Tally one child key read.
@@ -103,9 +106,11 @@ impl StateUsageStats {
self.reads.fetch_add(info.reads.ops, AtomicOrdering::Relaxed);
self.bytes_read.fetch_add(info.reads.bytes, AtomicOrdering::Relaxed);
self.writes_nodes.fetch_add(info.nodes_writes.ops, AtomicOrdering::Relaxed);
self.bytes_written_nodes.fetch_add(info.nodes_writes.bytes, AtomicOrdering::Relaxed);
self.bytes_written_nodes
.fetch_add(info.nodes_writes.bytes, AtomicOrdering::Relaxed);
self.removed_nodes.fetch_add(info.removed_nodes.ops, AtomicOrdering::Relaxed);
self.bytes_removed_nodes.fetch_add(info.removed_nodes.bytes, AtomicOrdering::Relaxed);
self.bytes_removed_nodes
.fetch_add(info.removed_nodes.bytes, AtomicOrdering::Relaxed);
self.reads_cache.fetch_add(info.cache_reads.ops, AtomicOrdering::Relaxed);
self.bytes_read_cache.fetch_add(info.cache_reads.bytes, AtomicOrdering::Relaxed);
}
File diff suppressed because it is too large Load Diff
+47 -23
View File
@@ -18,14 +18,16 @@
//! Database upgrade logic.
use std::fs;
use std::io::{Read, Write, ErrorKind};
use std::path::{Path, PathBuf};
use std::{
fs,
io::{ErrorKind, Read, Write},
path::{Path, PathBuf},
};
use sp_runtime::traits::Block as BlockT;
use crate::{columns, utils::DatabaseType};
use kvdb_rocksdb::{Database, DatabaseConfig};
use codec::{Decode, Encode};
use kvdb_rocksdb::{Database, DatabaseConfig};
use sp_runtime::traits::Block as BlockT;
/// Version file name.
const VERSION_FILE_NAME: &'static str = "db_version";
@@ -38,19 +40,28 @@ const V1_NUM_COLUMNS: u32 = 11;
const V2_NUM_COLUMNS: u32 = 12;
/// Upgrade database to current version.
pub fn upgrade_db<Block: BlockT>(db_path: &Path, db_type: DatabaseType) -> sp_blockchain::Result<()> {
pub fn upgrade_db<Block: BlockT>(
db_path: &Path,
db_type: DatabaseType,
) -> sp_blockchain::Result<()> {
let is_empty = db_path.read_dir().map_or(true, |mut d| d.next().is_none());
if !is_empty {
let db_version = current_version(db_path)?;
match db_version {
0 => Err(sp_blockchain::Error::Backend(format!("Unsupported database version: {}", db_version)))?,
0 => Err(sp_blockchain::Error::Backend(format!(
"Unsupported database version: {}",
db_version
)))?,
1 => {
migrate_1_to_2::<Block>(db_path, db_type)?;
migrate_2_to_3::<Block>(db_path, db_type)?
},
2 => migrate_2_to_3::<Block>(db_path, db_type)?,
CURRENT_VERSION => (),
_ => Err(sp_blockchain::Error::Backend(format!("Future database version: {}", db_version)))?,
_ => Err(sp_blockchain::Error::Backend(format!(
"Future database version: {}",
db_version
)))?,
}
}
@@ -60,8 +71,12 @@ pub fn upgrade_db<Block: BlockT>(db_path: &Path, db_type: DatabaseType) -> sp_bl
/// Migration from version1 to version2:
/// 1) the number of columns has changed from 11 to 12;
/// 2) transactions column is added;
fn migrate_1_to_2<Block: BlockT>(db_path: &Path, _db_type: DatabaseType) -> sp_blockchain::Result<()> {
let db_path = db_path.to_str()
fn migrate_1_to_2<Block: BlockT>(
db_path: &Path,
_db_type: DatabaseType,
) -> sp_blockchain::Result<()> {
let db_path = db_path
.to_str()
.ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?;
let db_cfg = DatabaseConfig::with_columns(V1_NUM_COLUMNS);
let db = Database::open(&db_cfg, db_path).map_err(db_err)?;
@@ -70,8 +85,12 @@ fn migrate_1_to_2<Block: BlockT>(db_path: &Path, _db_type: DatabaseType) -> sp_b
/// Migration from version2 to version3:
/// - The format of the stored Justification changed to support multiple Justifications.
fn migrate_2_to_3<Block: BlockT>(db_path: &Path, _db_type: DatabaseType) -> sp_blockchain::Result<()> {
let db_path = db_path.to_str()
fn migrate_2_to_3<Block: BlockT>(
db_path: &Path,
_db_type: DatabaseType,
) -> sp_blockchain::Result<()> {
let db_path = db_path
.to_str()
.ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?;
let db_cfg = DatabaseConfig::with_columns(V2_NUM_COLUMNS);
let db = Database::open(&db_cfg, db_path).map_err(db_err)?;
@@ -137,10 +156,11 @@ fn version_file_path(path: &Path) -> PathBuf {
#[cfg(test)]
mod tests {
use sc_state_db::PruningMode;
use crate::{DatabaseSettings, DatabaseSettingsSrc, KeepBlocks, TransactionStorageMode};
use crate::tests::Block;
use super::*;
use crate::{
tests::Block, DatabaseSettings, DatabaseSettingsSrc, KeepBlocks, TransactionStorageMode,
};
use sc_state_db::PruningMode;
fn create_db(db_path: &Path, version: Option<u32>) {
if let Some(version) = version {
@@ -151,14 +171,18 @@ mod tests {
}
fn open_database(db_path: &Path) -> sp_blockchain::Result<()> {
crate::utils::open_database::<Block>(&DatabaseSettings {
state_cache_size: 0,
state_cache_child_ratio: None,
state_pruning: PruningMode::ArchiveAll,
source: DatabaseSettingsSrc::RocksDb { path: db_path.to_owned(), cache_size: 128 },
keep_blocks: KeepBlocks::All,
transaction_storage: TransactionStorageMode::BlockBody,
}, DatabaseType::Full).map(|_| ())
crate::utils::open_database::<Block>(
&DatabaseSettings {
state_cache_size: 0,
state_cache_child_ratio: None,
state_pruning: PruningMode::ArchiveAll,
source: DatabaseSettingsSrc::RocksDb { path: db_path.to_owned(), cache_size: 128 },
keep_blocks: KeepBlocks::All,
transaction_storage: TransactionStorageMode::BlockBody,
},
DatabaseType::Full,
)
.map(|_| ())
}
#[test]

Some files were not shown because too many files have changed in this diff Show More