pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-13 01:11:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow updating configuration of changes tries (#3201)

Browse Source 
* DigestItem::ChangesTrieSignal

* introduce changes_trie::State

* introduce config activation block

* ChangesTrieSignal::as_new_configuration

* moved well_known_cache_keys to client

* extracted DbChangesTrieStorage to separate file

* change meaning of none in blockchain cache

* changes trie config (FULL) cache draft

* eliminating const ChangesTrieConfiguration

* delay pruning

* continue elimination

* do not prune CT config from cache

* removed redundant code

* fix some TODOs

* introduce ConfigurationRange

* use Configuration range in build

* build skewed digest

* remove debug print

* extracted surface iterator

* key_changes works with skewed digests

* fix client build

* add test for NeverPrune

* fix TODO

* fixed some TODOs

* more tests

* fixing TODOs

* fixed compilation

* update runtime version

* git rid of large tuple

* too long lines

* config_activation_block -> zero

* obsolete TODO

* removed unjustified expect

* update TODOs with issue number

* new CT pruning algorithm

fixed cache + multiple blocks finalization

track CT configuraiton on light clients

support CT configuration change revert

revert CT config test

new CT pruning algorithm

fixed cache + multiple blocks finalization

track CT configuraiton on light clients

support CT configuration change revert

revert CT config test

* BlockIdOrHeader isn't really required

* removed debug leftovers + some docs

* more docs

* more post-merge fixes

* more post-merge fixes

* revertes some unnecessary changes

* reverted unnecessary changes

* fix compilation + unnecessary changes

* (restart CI)

* fix cache update when finalizing multiple blocks

* fixed tests

* collect_extrinsics -> set_collect_extrinsics

* restore lost test

* do not calculate block number twice

* Update primitives/blockchain/src/error.rs

Co-Authored-By: cheme <emericchevalier.pro@gmail.com>

* map_err -> unwrap_or

* document get_at Result

* delete abandoned file

* added weight for set_changes_trie_config

* prefer_configs -> fail_if_disabled

* Update client/api/src/backend.rs

Co-Authored-By: cheme <emericchevalier.pro@gmail.com>

* Update client/db/src/changes_tries_storage.rs

Co-Authored-By: cheme <emericchevalier.pro@gmail.com>

* CommitOperation+merge -> CommitOperations

* fixed test compilation

* merged two different CTRange structs

* lost file

* uggrade db from v0 to v1 (init CT cache + add column)

* fix after merge

Co-authored-by: cheme <emericchevalier.pro@gmail.com>
Co-authored-by: Gavin Wood <github@gavwood.com>
This commit is contained in:
Svyatoslav Nikolsky
2020-01-16 19:38:24 +03:00
committed by Gavin Wood
parent 45fbf09dac
commit febf29390a
48 changed files with 2743 additions and 1548 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/client/api/src/backend.rs
+30 -11
View File
@@ -18,11 +18,11 @@
use std::sync::Arc;
use std::collections::HashMap;
use sp_core::ChangesTrieConfiguration;
use sp_core::ChangesTrieConfigurationRange;
use sp_core::offchain::OffchainStorage;
use sp_runtime::{generic::BlockId, Justification, Storage};
use sp_runtime::traits::{Block as BlockT, NumberFor, HasherFor};
use sp_state_machine::{ChangesTrieStorage as StateChangesTrieStorage, ChangesTrieTransaction};
use sp_state_machine::{ChangesTrieState, ChangesTrieStorage as StateChangesTrieStorage, ChangesTrieTransaction};
use crate::{
blockchain::{
Backend as BlockchainBackend, well_known_cache_keys
@@ -248,8 +248,6 @@ pub trait Backend<Block: BlockT>: AuxStore + Send + Sync {
type Blockchain: BlockchainBackend<Block>;
/// Associated state backend type.
type State: StateBackend<HasherFor<Block>> + Send;
/// Changes trie storage.
type ChangesTrieStorage: PrunableStateChangesTrieStorage<Block>;
/// Offchain workers local storage.
type OffchainStorage: OffchainStorage;
@@ -284,7 +282,7 @@ pub trait Backend<Block: BlockT>: AuxStore + Send + Sync {
fn usage_info(&self) -> Option<UsageInfo>;
/// Returns reference to changes trie storage.
fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage>;
fn changes_trie_storage(&self) -> Option<&dyn PrunableStateChangesTrieStorage<Block>>;
/// Returns a handle to offchain storage.
fn offchain_storage(&self) -> Option<Self::OffchainStorage>;
@@ -342,12 +340,16 @@ pub trait Backend<Block: BlockT>: AuxStore + Send + Sync {
pub trait PrunableStateChangesTrieStorage<Block: BlockT>:
StateChangesTrieStorage<HasherFor<Block>, NumberFor<Block>>
{
/// Get number block of oldest, non-pruned changes trie.
fn oldest_changes_trie_block(
&self,
config: &ChangesTrieConfiguration,
best_finalized: NumberFor<Block>,
) -> NumberFor<Block>;
/// Get reference to StateChangesTrieStorage.
fn storage(&self) -> &dyn StateChangesTrieStorage<HasherFor<Block>, NumberFor<Block>>;
/// Get configuration at given block.
fn configuration_at(&self, at: &BlockId<Block>) -> sp_blockchain::Result<
ChangesTrieConfigurationRange<NumberFor<Block>, Block::Hash>
>;
/// Get end block (inclusive) of oldest pruned max-level (or skewed) digest trie blocks range.
/// It is guaranteed that we have no any changes tries before (and including) this block.
/// It is guaranteed that all existing changes tries after this block are not yet pruned (if created).
fn oldest_pruned_digest_range_end(&self) -> NumberFor<Block>;
}
/// Mark for all Backend implementations, that are making use of state data, stored locally.
@@ -364,3 +366,20 @@ pub trait RemoteBackend<Block: BlockT>: Backend<Block> {
/// locally, or prepares request to fetch that data from remote node.
fn remote_blockchain(&self) -> Arc<dyn RemoteBlockchain<Block>>;
}
/// Return changes tries state at given block.
pub fn changes_tries_state_at_block<'a, Block: BlockT>(
block: &BlockId<Block>,
maybe_storage: Option<&'a dyn PrunableStateChangesTrieStorage<Block>>,
) -> sp_blockchain::Result<Option<ChangesTrieState<'a, HasherFor<Block>, NumberFor<Block>>>> {
let storage = match maybe_storage {
Some(storage) => storage,
None => return Ok(None),
};
let config_range = storage.configuration_at(block)?;
match config_range.config {
Some(config) => Ok(Some(ChangesTrieState::new(config, config_range.zero.0, storage.storage()))),
None => Ok(None),
}
}
substrate/client/api/src/light.rs
+3 -3
View File
@@ -26,7 +26,7 @@ use sp_runtime::{
},
generic::BlockId
};
use sp_core::ChangesTrieConfiguration;
use sp_core::ChangesTrieConfigurationRange;
use sp_state_machine::StorageProof;
use sp_blockchain::{
HeaderMetadata, well_known_cache_keys, HeaderBackend, Cache as BlockchainCache,
@@ -96,8 +96,8 @@ pub struct RemoteReadChildRequest<Header: HeaderT> {
/// Remote key changes read request.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RemoteChangesRequest<Header: HeaderT> {
/// Changes trie configuration.
pub changes_trie_config: ChangesTrieConfiguration,
/// All changes trie configurations that are valid within [first_block; last_block].
pub changes_trie_configs: Vec<ChangesTrieConfigurationRange<Header::Number, Header::Hash>>,
/// Query changes from range of blocks, starting (and including) with this hash...
pub first_block: (Header::Number, Header::Hash),
/// ...ending (and including) with this hash. Should come after first_block and
substrate/client/authority-discovery/src/tests.rs
+2 -4
View File
@@ -212,12 +212,10 @@ impl ApiExt<Block> for RuntimeApi {
unimplemented!("Not required for testing!")
}
fn into_storage_changes<
T: sp_api::ChangesTrieStorage<sp_api::HasherFor<Block>, sp_api::NumberFor<Block>>
>(
fn into_storage_changes(
&self,
_: &Self::StateBackend,
_: Option<&T>,
_: Option<&sp_api::ChangesTrieState<sp_api::HasherFor<Block>, sp_api::NumberFor<Block>>>,
_: <Block as sp_api::BlockT>::Hash,
) -> std::result::Result<sp_api::StorageChanges<Self::StateBackend, Block>, String>
where Self: Sized
substrate/client/basic-authorship/src/basic_authorship.rs
+5 -2
View File
@@ -374,9 +374,12 @@ mod tests {
api.execute_block(&block_id, proposal.block).unwrap();
let state = backend.state_at(block_id).unwrap();
let changes_trie_storage = backend.changes_trie_storage();
let changes_trie_state = backend::changes_tries_state_at_block(
&block_id,
backend.changes_trie_storage(),
).unwrap();
let storage_changes = api.into_storage_changes(&state, changes_trie_storage, genesis_hash)
let storage_changes = api.into_storage_changes(&state, changes_trie_state.as_ref(), genesis_hash)
.unwrap();
assert_eq!(
substrate/client/block-builder/src/lib.rs
+5 -2
View File
@@ -191,14 +191,17 @@ where
let proof = self.api.extract_proof();
let state = self.backend.state_at(self.block_id)?;
let changes_trie_storage = self.backend.changes_trie_storage();
let changes_trie_state = backend::changes_tries_state_at_block(
&self.block_id,
self.backend.changes_trie_storage(),
)?;
let parent_hash = self.parent_hash;
// The unsafe is required because the consume requires that we drop/consume the inner api
// (what we do here).
let storage_changes = self.api.into_storage_changes(
&state,
changes_trie_storage,
changes_trie_state.as_ref(),
parent_hash,
);
substrate/client/consensus/aura/src/lib.rs
+11 -8
View File
@@ -675,19 +675,21 @@ fn initialize_authorities_cache<A, B, C>(client: &C) -> Result<(), ConsensusErro
};
// check if we already have initialized the cache
let genesis_id = BlockId::Number(Zero::zero());
let genesis_authorities: Option<Vec<A>> = cache
.get_at(&well_known_cache_keys::AUTHORITIES, &genesis_id)
.and_then(|(_, _, v)| Decode::decode(&mut &v[..]).ok());
if genesis_authorities.is_some() {
return Ok(());
}
let map_err = |error| sp_consensus::Error::from(sp_consensus::Error::ClientImport(
format!(
"Error initializing authorities cache: {}",
error,
)));
let genesis_id = BlockId::Number(Zero::zero());
let genesis_authorities: Option<Vec<A>> = cache
.get_at(&well_known_cache_keys::AUTHORITIES, &genesis_id)
.unwrap_or(None)
.and_then(|(_, _, v)| Decode::decode(&mut &v[..]).ok());
if genesis_authorities.is_some() {
return Ok(());
}
let genesis_authorities = authorities(client, &genesis_id)?;
cache.initialize(&well_known_cache_keys::AUTHORITIES, genesis_authorities.encode())
.map_err(map_err)?;
@@ -706,6 +708,7 @@ fn authorities<A, B, C>(client: &C, at: &BlockId<B>) -> Result<Vec<A>, Consensus
.cache()
.and_then(|cache| cache
.get_at(&well_known_cache_keys::AUTHORITIES, at)
.unwrap_or(None)
.and_then(|(_, _, v)| Decode::decode(&mut &v[..]).ok())
)
.or_else(|| AuraApi::authorities(&*client.runtime_api(), at).ok())
substrate/client/db/Cargo.toml
+2
View File
@@ -31,6 +31,8 @@ sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" }
substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" }
env_logger = "0.7.0"
quickcheck = "0.9"
kvdb-rocksdb = "0.4"
tempdir = "0.3"
[features]
default = []
substrate/client/db/src/cache/list_cache.rs
Vendored
+392 -157
View File
@@ -94,6 +94,12 @@ pub enum CommitOperation<Block: BlockT, T: CacheItemT> {
BlockReverted(BTreeMap<usize, Option<Fork<Block, T>>>),
}
/// A set of commit operations.
#[derive(Debug)]
pub struct CommitOperations<Block: BlockT, T: CacheItemT> {
operations: Vec<CommitOperation<Block, T>>,
}
/// Single fork of list-based cache.
#[derive(Debug)]
#[cfg_attr(test, derive(PartialEq))]
@@ -123,21 +129,17 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
storage: S,
pruning_strategy: PruningStrategy<NumberFor<Block>>,
best_finalized_block: ComplexBlockId<Block>,
) -> Self {
) -> ClientResult<Self> {
let (best_finalized_entry, unfinalized) = storage.read_meta()
.and_then(|meta| read_forks(&storage, meta))
.unwrap_or_else(|error| {
warn!(target: "db", "Unable to initialize list cache: {}. Restarting", error);
(None, Vec::new())
});
.and_then(|meta| read_forks(&storage, meta))?;
ListCache {
Ok(ListCache {
storage,
pruning_strategy,
best_finalized_block,
best_finalized_entry,
unfinalized,
}
})
}
/// Get reference to the storage.
@@ -145,6 +147,12 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
&self.storage
}
/// Get unfinalized forks reference.
#[cfg(test)]
pub fn unfinalized(&self) -> &[Fork<Block, T>] {
&self.unfinalized
}
/// Get value valid at block.
pub fn value_at_block(
&self,
@@ -156,8 +164,8 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
// BUT since we're not guaranteeing to provide correct values for forks
// behind the finalized block, check if the block is finalized first
if !chain::is_finalized_block(&self.storage, at, Bounded::max_value())? {
return Ok(None);
if !chain::is_finalized_block(&self.storage, &at, Bounded::max_value())? {
return Err(ClientError::NotInFinalizedChain);
}
self.best_finalized_entry.as_ref()
@@ -171,11 +179,14 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
// IF there's no matching fork, ensure that this isn't a block from a fork that has forked
// behind the best finalized block and search at finalized fork
match self.find_unfinalized_fork(at)? {
match self.find_unfinalized_fork(&at)? {
Some(fork) => Some(&fork.head),
None => match self.best_finalized_entry.as_ref() {
Some(best_finalized_entry) if chain::is_connected_to_block(&self.storage, &best_finalized_entry.valid_from, at)? =>
Some(best_finalized_entry),
Some(best_finalized_entry) if chain::is_connected_to_block(
&self.storage,
&at,
&best_finalized_entry.valid_from,
)? => Some(best_finalized_entry),
_ => None,
},
}
@@ -198,9 +209,98 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
block: ComplexBlockId<Block>,
value: Option<T>,
entry_type: EntryType,
operations: &mut CommitOperations<Block, T>,
) -> ClientResult<()> {
Ok(operations.append(self.do_on_block_insert(tx, parent, block, value, entry_type, operations)?))
}
/// When previously inserted block is finalized.
pub fn on_block_finalize<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>,
operations: &mut CommitOperations<Block, T>,
) -> ClientResult<()> {
Ok(operations.append(self.do_on_block_finalize(tx, parent, block, operations)?))
}
/// When block is reverted.
pub fn on_block_revert<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
reverted_block: &ComplexBlockId<Block>,
operations: &mut CommitOperations<Block, T>,
) -> ClientResult<()> {
Ok(operations.append(Some(self.do_on_block_revert(tx, reverted_block)?)))
}
/// When transaction is committed.
pub fn on_transaction_commit(&mut self, ops: CommitOperations<Block, T>) {
for op in ops.operations {
match op {
CommitOperation::AppendNewBlock(index, best_block) => {
let mut fork = self.unfinalized.get_mut(index)
.expect("ListCache is a crate-private type;
internal clients of ListCache are committing transaction while cache is locked;
CommitOperation holds valid references while cache is locked; qed");
fork.best_block = Some(best_block);
},
CommitOperation::AppendNewEntry(index, entry) => {
let mut fork = self.unfinalized.get_mut(index)
.expect("ListCache is a crate-private type;
internal clients of ListCache are committing transaction while cache is locked;
CommitOperation holds valid references while cache is locked; qed");
fork.best_block = Some(entry.valid_from.clone());
fork.head = entry;
},
CommitOperation::AddNewFork(entry) => {
self.unfinalized.push(Fork {
best_block: Some(entry.valid_from.clone()),
head: entry,
});
},
CommitOperation::BlockFinalized(block, finalizing_entry, forks) => {
self.best_finalized_block = block;
if let Some(finalizing_entry) = finalizing_entry {
self.best_finalized_entry = Some(finalizing_entry);
}
for fork_index in forks.iter().rev() {
self.unfinalized.remove(*fork_index);
}
},
CommitOperation::BlockReverted(forks) => {
for (fork_index, updated_fork) in forks.into_iter().rev() {
match updated_fork {
Some(updated_fork) => self.unfinalized[fork_index] = updated_fork,
None => { self.unfinalized.remove(fork_index); },
}
}
},
}
}
}
fn do_on_block_insert<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>,
value: Option<T>,
entry_type: EntryType,
operations: &CommitOperations<Block, T>,
) -> ClientResult<Option<CommitOperation<Block, T>>> {
// this guarantee is currently provided by LightStorage && we're relying on it here
debug_assert!(entry_type != EntryType::Final || self.best_finalized_block.hash == parent.hash);
let prev_operation = operations.operations.last();
debug_assert!(
entry_type != EntryType::Final ||
self.best_finalized_block.hash == parent.hash ||
match prev_operation {
Some(&CommitOperation::BlockFinalized(ref best_finalized_block, _, _))
=> best_finalized_block.hash == parent.hash,
_ => false,
}
);
// we do not store any values behind finalized
if block.number != Zero::zero() && self.best_finalized_block.number >= block.number {
@@ -296,7 +396,7 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
}
// cleanup database from abandoned unfinalized forks and obsolete finalized entries
let abandoned_forks = self.destroy_abandoned_forks(tx, &block);
let abandoned_forks = self.destroy_abandoned_forks(tx, &block, prev_operation);
self.prune_finalized_entries(tx, &block);
match new_storage_entry {
@@ -310,34 +410,39 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
}
}
/// When previously inserted block is finalized.
pub fn on_block_finalize<Tx: StorageTransaction<Block, T>>(
fn do_on_block_finalize<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>,
operations: &CommitOperations<Block, T>,
) -> ClientResult<Option<CommitOperation<Block, T>>> {
// this guarantee is currently provided by LightStorage && we're relying on it here
debug_assert_eq!(self.best_finalized_block.hash, parent.hash);
// this guarantee is currently provided by db backend && we're relying on it here
let prev_operation = operations.operations.last();
debug_assert!(
self.best_finalized_block.hash == parent.hash ||
match prev_operation {
Some(&CommitOperation::BlockFinalized(ref best_finalized_block, _, _))
=> best_finalized_block.hash == parent.hash,
_ => false,
}
);
// there could be at most one entry that is finalizing
let finalizing_entry = self.storage.read_entry(&block)?
.map(|entry| entry.into_entry(block.clone()));
// cleanup database from abandoned unfinalized forks and obsolete finalized entries
let abandoned_forks = self.destroy_abandoned_forks(tx, &block);
let abandoned_forks = self.destroy_abandoned_forks(tx, &block, prev_operation);
self.prune_finalized_entries(tx, &block);
let update_meta = finalizing_entry.is_some();
let operation = CommitOperation::BlockFinalized(block, finalizing_entry, abandoned_forks);
if update_meta {
tx.update_meta(self.best_finalized_entry.as_ref(), &self.unfinalized, &operation);
}
tx.update_meta(self.best_finalized_entry.as_ref(), &self.unfinalized, &operation);
Ok(Some(operation))
}
/// When block is reverted.
pub fn on_block_revert<Tx: StorageTransaction<Block, T>>(
fn do_on_block_revert<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
reverted_block: &ComplexBlockId<Block>,
@@ -374,50 +479,6 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
Ok(operation)
}
/// When transaction is committed.
pub fn on_transaction_commit(&mut self, op: CommitOperation<Block, T>) {
match op {
CommitOperation::AppendNewBlock(index, best_block) => {
let mut fork = self.unfinalized.get_mut(index)
.expect("ListCache is a crate-private type;
internal clients of ListCache are committing transaction while cache is locked;
CommitOperation holds valid references while cache is locked; qed");
fork.best_block = Some(best_block);
},
CommitOperation::AppendNewEntry(index, entry) => {
let mut fork = self.unfinalized.get_mut(index)
.expect("ListCache is a crate-private type;
internal clients of ListCache are committing transaction while cache is locked;
CommitOperation holds valid references while cache is locked; qed");
fork.best_block = Some(entry.valid_from.clone());
fork.head = entry;
},
CommitOperation::AddNewFork(entry) => {
self.unfinalized.push(Fork {
best_block: Some(entry.valid_from.clone()),
head: entry,
});
},
CommitOperation::BlockFinalized(block, finalizing_entry, forks) => {
self.best_finalized_block = block;
if let Some(finalizing_entry) = finalizing_entry {
self.best_finalized_entry = Some(finalizing_entry);
}
for fork_index in forks.iter().rev() {
self.unfinalized.remove(*fork_index);
}
},
CommitOperation::BlockReverted(forks) => {
for (fork_index, updated_fork) in forks.into_iter().rev() {
match updated_fork {
Some(updated_fork) => self.unfinalized[fork_index] = updated_fork,
None => { self.unfinalized.remove(fork_index); },
}
}
},
}
}
/// Prune old finalized entries.
fn prune_finalized_entries<Tx: StorageTransaction<Block, T>>(
&self,
@@ -475,10 +536,22 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
fn destroy_abandoned_forks<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
block: &ComplexBlockId<Block>
block: &ComplexBlockId<Block>,
prev_operation: Option<&CommitOperation<Block, T>>,
) -> BTreeSet<usize> {
let mut destroyed = BTreeSet::new();
for (index, fork) in self.unfinalized.iter().enumerate() {
// if some block has been finalized already => take it into account
let prev_abandoned_forks = match prev_operation {
Some(&CommitOperation::BlockFinalized(_, _, ref abandoned_forks)) => Some(abandoned_forks),
_ => None,
};
let mut destroyed = prev_abandoned_forks.cloned().unwrap_or_else(|| BTreeSet::new());
let live_unfinalized = self.unfinalized.iter()
.enumerate()
.filter(|(idx, _)| prev_abandoned_forks
.map(|prev_abandoned_forks| !prev_abandoned_forks.contains(idx))
.unwrap_or(true));
for (index, fork) in live_unfinalized {
if fork.head.valid_from.number == block.number {
destroyed.insert(index);
if fork.head.valid_from.hash != block.hash {
@@ -493,7 +566,10 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
}
/// Search unfinalized fork where given block belongs.
fn find_unfinalized_fork(&self, block: &ComplexBlockId<Block>) -> ClientResult<Option<&Fork<Block, T>>> {
fn find_unfinalized_fork(
&self,
block: &ComplexBlockId<Block>,
) -> ClientResult<Option<&Fork<Block, T>>> {
for unfinalized in &self.unfinalized {
if unfinalized.matches(&self.storage, block)? {
return Ok(Some(&unfinalized));
@@ -549,7 +625,7 @@ impl<Block: BlockT, T: CacheItemT> Fork<Block, T> {
};
// check if the parent is connected to the beginning of the range
if !chain::is_connected_to_block(storage, &parent, &begin)? {
if !chain::is_connected_to_block(storage, parent, &begin)? {
return Ok(None);
}
@@ -621,6 +697,65 @@ impl<Block: BlockT, T: CacheItemT> Fork<Block, T> {
}
}
impl<Block: BlockT, T: CacheItemT> Default for CommitOperations<Block, T> {
fn default() -> Self {
CommitOperations { operations: Vec::new() }
}
}
// This should never be allowed for non-test code to avoid revealing its internals.
#[cfg(test)]
impl<Block: BlockT, T: CacheItemT> From<Vec<CommitOperation<Block, T>>> for CommitOperations<Block, T> {
fn from(operations: Vec<CommitOperation<Block, T>>) -> Self {
CommitOperations { operations }
}
}
impl<Block: BlockT, T: CacheItemT> CommitOperations<Block, T> {
/// Append operation to the set.
fn append(&mut self, new_operation: Option<CommitOperation<Block, T>>) {
let new_operation = match new_operation {
Some(new_operation) => new_operation,
None => return,
};
let last_operation = match self.operations.pop() {
Some(last_operation) => last_operation,
None => {
self.operations.push(new_operation);
return;
},
};
// we are able (and obliged to) to merge two consequent block finalization operations
match last_operation {
CommitOperation::BlockFinalized(old_finalized_block, old_finalized_entry, old_abandoned_forks) => {
match new_operation {
CommitOperation::BlockFinalized(new_finalized_block, new_finalized_entry, new_abandoned_forks) => {
self.operations.push(CommitOperation::BlockFinalized(
new_finalized_block,
new_finalized_entry,
new_abandoned_forks,
));
},
_ => {
self.operations.push(CommitOperation::BlockFinalized(
old_finalized_block,
old_finalized_entry,
old_abandoned_forks,
));
self.operations.push(new_operation);
},
}
},
_ => {
self.operations.push(last_operation);
self.operations.push(new_operation);
},
}
}
}
/// Destroy fork by deleting all unfinalized entries.
pub fn destroy_fork<Block: BlockT, T: CacheItemT, S: Storage<Block, T>, Tx: StorageTransaction<Block, T>>(
head_valid_from: ComplexBlockId<Block>,
@@ -674,7 +809,7 @@ mod chain {
block1: &ComplexBlockId<Block>,
block2: &ComplexBlockId<Block>,
) -> ClientResult<bool> {
let (begin, end) = if block1 > block2 { (block2, block1) } else { (block1, block2) };
let (begin, end) = if *block1 > *block2 { (block2, block1) } else { (block1, block2) };
let mut current = storage.read_header(&end.hash)?
.ok_or_else(|| ClientError::UnknownBlock(format!("{}", end.hash)))?;
while *current.number() > begin.number {
@@ -773,8 +908,8 @@ pub mod tests {
// when block is earlier than best finalized block AND it is not finalized
// --- 50 ---
// ----------> [100]
assert_eq!(ListCache::<_, u64, _>::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
.value_at_block(&test_id(50)).unwrap(), None);
assert!(ListCache::<_, u64, _>::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
.unwrap().value_at_block(&test_id(50)).is_err());
// when block is earlier than best finalized block AND it is finalized AND value is some
// [30] ---- 50 ---> [100]
assert_eq!(ListCache::new(
@@ -784,7 +919,7 @@ pub mod tests {
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 })
.with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }),
PruningStrategy::ByDepth(1024), test_id(100)
).value_at_block(&test_id(50)).unwrap(), Some((test_id(30), Some(test_id(100)), 30)));
).unwrap().value_at_block(&test_id(50)).unwrap(), Some((test_id(30), Some(test_id(100)), 30)));
// when block is the best finalized block AND value is some
// ---> [100]
assert_eq!(ListCache::new(
@@ -794,18 +929,18 @@ pub mod tests {
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 })
.with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }),
PruningStrategy::ByDepth(1024), test_id(100)
).value_at_block(&test_id(100)).unwrap(), Some((test_id(100), None, 100)));
).unwrap().value_at_block(&test_id(100)).unwrap(), Some((test_id(100), None, 100)));
// when block is parallel to the best finalized block
// ---- 100
// ---> [100]
assert_eq!(ListCache::new(
assert!(ListCache::new(
DummyStorage::new()
.with_meta(Some(test_id(100)), Vec::new())
.with_id(50, H256::from_low_u64_be(50))
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 })
.with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }),
PruningStrategy::ByDepth(1024), test_id(100)
).value_at_block(&ComplexBlockId::new(H256::from_low_u64_be(2), 100)).unwrap(), None);
).unwrap().value_at_block(&ComplexBlockId::new(H256::from_low_u64_be(2), 100)).is_err());
// when block is later than last finalized block AND there are no forks AND finalized value is Some
// ---> [100] --- 200
@@ -815,7 +950,7 @@ pub mod tests {
.with_id(50, H256::from_low_u64_be(50))
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 }),
PruningStrategy::ByDepth(1024), test_id(100)
).value_at_block(&test_id(200)).unwrap(), Some((test_id(100), None, 100)));
).unwrap().value_at_block(&test_id(200)).unwrap(), Some((test_id(100), None, 100)));
// when block is later than last finalized block AND there are no matching forks
// AND block is connected to finalized block AND finalized value is Some
@@ -831,7 +966,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(fork_header(0, 2, 3)),
PruningStrategy::ByDepth(1024), test_id(2)
).value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
).unwrap().value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
// when block is later than last finalized block AND there are no matching forks
// AND block is not connected to finalized block
// --- 2 --- 3
@@ -848,7 +983,7 @@ pub mod tests {
.with_header(fork_header(0, 1, 3))
.with_header(fork_header(0, 1, 2)),
PruningStrategy::ByDepth(1024), test_id(2)
).value_at_block(&fork_id(0, 1, 3)).unwrap(), None);
).unwrap().value_at_block(&fork_id(0, 1, 3)).unwrap(), None);
// when block is later than last finalized block AND it appends to unfinalized fork from the end
// AND unfinalized value is Some
@@ -861,7 +996,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(test_header(5)),
PruningStrategy::ByDepth(1024), test_id(2)
).value_at_block(&correct_id(5)).unwrap(), Some((correct_id(4), None, 4)));
).unwrap().value_at_block(&correct_id(5)).unwrap(), Some((correct_id(4), None, 4)));
// when block is later than last finalized block AND it does not fits unfinalized fork
// AND it is connected to the finalized block AND finalized value is Some
// ---> [2] ----------> [4]
@@ -876,7 +1011,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(fork_header(0, 2, 3)),
PruningStrategy::ByDepth(1024), test_id(2)
).value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
).unwrap().value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
}
#[test]
@@ -885,22 +1020,25 @@ pub mod tests {
let fin = EntryType::Final;
// when trying to insert block < finalized number
assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
.on_block_insert(
let mut ops = Default::default();
assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100)).unwrap()
.do_on_block_insert(
&mut DummyTransaction::new(),
test_id(49),
test_id(50),
Some(50),
nfin,
&mut ops,
).unwrap().is_none());
// when trying to insert block @ finalized number
assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
.on_block_insert(
assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100)).unwrap()
.do_on_block_insert(
&mut DummyTransaction::new(),
test_id(99),
test_id(100),
Some(100),
nfin,
&Default::default(),
).unwrap().is_none());
// when trying to insert non-final block AND it appends to the best block of unfinalized fork
@@ -910,19 +1048,23 @@ pub mod tests {
.with_meta(None, vec![test_id(4)])
.with_entry(test_id(4), StorageEntry { prev_valid_from: None, value: 4 }),
PruningStrategy::ByDepth(1024), test_id(2)
);
).unwrap();
cache.unfinalized[0].best_block = Some(test_id(4));
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, test_id(4), test_id(5), Some(4), nfin).unwrap(),
Some(CommitOperation::AppendNewBlock(0, test_id(5))));
assert_eq!(
cache.do_on_block_insert(&mut tx, test_id(4), test_id(5), Some(4), nfin, &Default::default()).unwrap(),
Some(CommitOperation::AppendNewBlock(0, test_id(5))),
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
// when trying to insert non-final block AND it appends to the best block of unfinalized fork
// AND new value is the same as in the fork' best block
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, test_id(4), test_id(5), Some(5), nfin).unwrap(),
Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: test_id(5), value: 5 })));
assert_eq!(
cache.do_on_block_insert(&mut tx, test_id(4), test_id(5), Some(5), nfin, &Default::default()).unwrap(),
Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: test_id(5), value: 5 })),
);
assert_eq!(*tx.inserted_entries(), vec![test_id(5).hash].into_iter().collect());
assert!(tx.removed_entries().is_empty());
assert_eq!(*tx.updated_meta(), Some(Metadata { finalized: None, unfinalized: vec![test_id(5)] }));
@@ -935,18 +1077,36 @@ pub mod tests {
.with_entry(correct_id(4), StorageEntry { prev_valid_from: None, value: 4 })
.with_header(test_header(4)),
PruningStrategy::ByDepth(1024), test_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(4), correct_id(5), Some(4), nfin).unwrap(),
Some(CommitOperation::AppendNewBlock(0, correct_id(5))));
assert_eq!(
cache.do_on_block_insert(
&mut tx,
correct_id(4),
correct_id(5),
Some(4),
nfin,
&Default::default(),
).unwrap(),
Some(CommitOperation::AppendNewBlock(0, correct_id(5))),
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
// when trying to insert non-final block AND it is the first block that appends to the best block of unfinalized fork
// AND new value is the same as in the fork' best block
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(4), correct_id(5), Some(5), nfin).unwrap(),
Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(5), value: 5 })));
assert_eq!(
cache.do_on_block_insert(
&mut tx,
correct_id(4),
correct_id(5),
Some(5),
nfin,
&Default::default(),
).unwrap(),
Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(5), value: 5 })),
);
assert_eq!(*tx.inserted_entries(), vec![correct_id(5).hash].into_iter().collect());
assert!(tx.removed_entries().is_empty());
assert_eq!(*tx.updated_meta(), Some(Metadata { finalized: None, unfinalized: vec![correct_id(5)] }));
@@ -961,10 +1121,13 @@ pub mod tests {
.with_header(test_header(3))
.with_header(test_header(4)),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(3), fork_id(0, 3, 4), Some(14), nfin).unwrap(),
Some(CommitOperation::AddNewFork(Entry { valid_from: fork_id(0, 3, 4), value: 14 })));
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(3), fork_id(0, 3, 4), Some(14), nfin, &Default::default())
.unwrap(),
Some(CommitOperation::AddNewFork(Entry { valid_from: fork_id(0, 3, 4), value: 14 })),
);
assert_eq!(*tx.inserted_entries(), vec![fork_id(0, 3, 4).hash].into_iter().collect());
assert!(tx.removed_entries().is_empty());
assert_eq!(*tx.updated_meta(), Some(Metadata { finalized: Some(correct_id(2)), unfinalized: vec![correct_id(4), fork_id(0, 3, 4)] }));
@@ -976,9 +1139,13 @@ pub mod tests {
.with_meta(Some(correct_id(2)), vec![])
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), nfin).unwrap(), None);
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), nfin, &Default::default())
.unwrap(),
None,
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
@@ -989,19 +1156,23 @@ pub mod tests {
.with_meta(Some(correct_id(2)), vec![])
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), nfin).unwrap(),
Some(CommitOperation::AddNewFork(Entry { valid_from: correct_id(3), value: 3 })));
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), nfin, &Default::default())
.unwrap(),
Some(CommitOperation::AddNewFork(Entry { valid_from: correct_id(3), value: 3 })),
);
assert_eq!(*tx.inserted_entries(), vec![correct_id(3).hash].into_iter().collect());
assert!(tx.removed_entries().is_empty());
assert_eq!(*tx.updated_meta(), Some(Metadata { finalized: Some(correct_id(2)), unfinalized: vec![correct_id(3)] }));
// when inserting finalized entry AND there are no previous finalized entries
let cache = ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), correct_id(2));
let cache = ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), correct_id(2)).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(
cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), fin).unwrap(),
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), fin, &Default::default())
.unwrap(),
Some(CommitOperation::BlockFinalized(
correct_id(3),
Some(Entry { valid_from: correct_id(3), value: 3 }),
@@ -1017,17 +1188,19 @@ pub mod tests {
.with_meta(Some(correct_id(2)), vec![])
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())));
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin, &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())),
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
// when inserting finalized entry AND value differs from previous finalized
let mut tx = DummyTransaction::new();
assert_eq!(
cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), fin).unwrap(),
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), fin, &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(
correct_id(3),
Some(Entry { valid_from: correct_id(3), value: 3 }),
@@ -1045,10 +1218,12 @@ pub mod tests {
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
.with_entry(fork_id(0, 1, 3), StorageEntry { prev_valid_from: None, value: 13 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, vec![0].into_iter().collect())));
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin, &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, vec![0].into_iter().collect())),
);
}
#[test]
@@ -1060,13 +1235,18 @@ pub mod tests {
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_finalize(&mut tx, correct_id(2), correct_id(3)).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())));
assert_eq!(
cache.do_on_block_finalize(&mut tx, correct_id(2), correct_id(3), &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())),
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
assert_eq!(
*tx.updated_meta(),
Some(Metadata { finalized: Some(correct_id(2)), unfinalized: vec![correct_id(5)] }),
);
// finalization finalizes entry
let cache = ListCache::new(
DummyStorage::new()
@@ -1074,10 +1254,10 @@ pub mod tests {
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 }),
PruningStrategy::ByDepth(1024), correct_id(4)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(
cache.on_block_finalize(&mut tx, correct_id(4), correct_id(5)).unwrap(),
cache.do_on_block_finalize(&mut tx, correct_id(4), correct_id(5), &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(
correct_id(5),
Some(Entry { valid_from: correct_id(5), value: 5 }),
@@ -1094,10 +1274,12 @@ pub mod tests {
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
.with_entry(fork_id(0, 1, 3), StorageEntry { prev_valid_from: None, value: 13 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_finalize(&mut tx, correct_id(2), correct_id(3)).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, vec![0].into_iter().collect())));
assert_eq!(
cache.do_on_block_finalize(&mut tx, correct_id(2), correct_id(3), &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, vec![0].into_iter().collect())),
);
}
#[test]
@@ -1109,25 +1291,29 @@ pub mod tests {
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 })
.with_entry(correct_id(6), StorageEntry { prev_valid_from: Some(correct_id(5)), value: 6 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
// when new block is appended to unfinalized fork
cache.on_transaction_commit(CommitOperation::AppendNewBlock(0, correct_id(6)));
cache.on_transaction_commit(vec![CommitOperation::AppendNewBlock(0, correct_id(6))].into());
assert_eq!(cache.unfinalized[0].best_block, Some(correct_id(6)));
// when new entry is appended to unfinalized fork
cache.on_transaction_commit(CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(7), value: 7 }));
cache.on_transaction_commit(vec![
CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(7), value: 7 }),
].into());
assert_eq!(cache.unfinalized[0].best_block, Some(correct_id(7)));
assert_eq!(cache.unfinalized[0].head, Entry { valid_from: correct_id(7), value: 7 });
// when new fork is added
cache.on_transaction_commit(CommitOperation::AddNewFork(Entry { valid_from: correct_id(10), value: 10 }));
cache.on_transaction_commit(vec![
CommitOperation::AddNewFork(Entry { valid_from: correct_id(10), value: 10 }),
].into());
assert_eq!(cache.unfinalized[2].best_block, Some(correct_id(10)));
assert_eq!(cache.unfinalized[2].head, Entry { valid_from: correct_id(10), value: 10 });
// when block is finalized + entry is finalized + unfinalized forks are deleted
cache.on_transaction_commit(CommitOperation::BlockFinalized(
cache.on_transaction_commit(vec![CommitOperation::BlockFinalized(
correct_id(20),
Some(Entry { valid_from: correct_id(20), value: 20 }),
vec![0, 1, 2].into_iter().collect(),
));
)].into());
assert_eq!(cache.best_finalized_block, correct_id(20));
assert_eq!(cache.best_finalized_entry, Some(Entry { valid_from: correct_id(20), value: 20 }));
assert!(cache.unfinalized.is_empty());
@@ -1148,7 +1334,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(test_header(5)),
PruningStrategy::ByDepth(1024), correct_id(0)
).find_unfinalized_fork(&correct_id(4)).unwrap().unwrap().head.valid_from, correct_id(5));
).unwrap().find_unfinalized_fork((&correct_id(4)).into()).unwrap().unwrap().head.valid_from, correct_id(5));
// --- [2] ---------------> [5]
// ----------> [3] ---> 4
assert_eq!(ListCache::new(
@@ -1165,7 +1351,8 @@ pub mod tests {
.with_header(fork_header(0, 1, 3))
.with_header(fork_header(0, 1, 4)),
PruningStrategy::ByDepth(1024), correct_id(0)
).find_unfinalized_fork(&fork_id(0, 1, 4)).unwrap().unwrap().head.valid_from, fork_id(0, 1, 3));
).unwrap()
.find_unfinalized_fork((&fork_id(0, 1, 4)).into()).unwrap().unwrap().head.valid_from, fork_id(0, 1, 3));
// --- [2] ---------------> [5]
// ----------> [3]
// -----------------> 4
@@ -1185,7 +1372,7 @@ pub mod tests {
.with_header(fork_header(1, 1, 3))
.with_header(fork_header(1, 1, 4)),
PruningStrategy::ByDepth(1024), correct_id(0)
).find_unfinalized_fork(&fork_id(1, 1, 4)).unwrap().is_none());
).unwrap().find_unfinalized_fork((&fork_id(1, 1, 4)).into()).unwrap().is_none());
}
#[test]
@@ -1195,7 +1382,7 @@ pub mod tests {
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 })
.with_entry(test_id(50), StorageEntry { prev_valid_from: None, value: 50 });
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: test_id(100), value: 0 } }
.matches(&storage, &test_id(20)).unwrap(), false);
.matches(&storage, (&test_id(20)).into()).unwrap(), false);
// when block is not connected to the begin block
let storage = DummyStorage::new()
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 })
@@ -1206,7 +1393,7 @@ pub mod tests {
.with_header(fork_header(0, 2, 4))
.with_header(fork_header(0, 2, 3));
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: correct_id(5), value: 100 } }
.matches(&storage, &fork_id(0, 2, 4)).unwrap(), false);
.matches(&storage, (&fork_id(0, 2, 4)).into()).unwrap(), false);
// when block is not connected to the end block
let storage = DummyStorage::new()
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 })
@@ -1216,14 +1403,14 @@ pub mod tests {
.with_header(test_header(3))
.with_header(fork_header(0, 3, 4));
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: correct_id(5), value: 100 } }
.matches(&storage, &fork_id(0, 3, 4)).unwrap(), false);
.matches(&storage, (&fork_id(0, 3, 4)).into()).unwrap(), false);
// when block is connected to the begin block AND end is open
let storage = DummyStorage::new()
.with_entry(correct_id(5), StorageEntry { prev_valid_from: None, value: 100 })
.with_header(test_header(5))
.with_header(test_header(6));
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: correct_id(5), value: 100 } }
.matches(&storage, &correct_id(6)).unwrap(), true);
.matches(&storage, (&correct_id(6)).into()).unwrap(), true);
// when block is connected to the begin block AND to the end block
let storage = DummyStorage::new()
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 })
@@ -1232,7 +1419,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(test_header(3));
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: correct_id(5), value: 100 } }
.matches(&storage, &correct_id(4)).unwrap(), true);
.matches(&storage, (&correct_id(4)).into()).unwrap(), true);
}
#[test]
@@ -1338,9 +1525,21 @@ pub mod tests {
#[test]
fn is_connected_to_block_fails() {
// when storage returns error
assert!(chain::is_connected_to_block::<_, u64, _>(&FaultyStorage, &test_id(1), &test_id(100)).is_err());
assert!(
chain::is_connected_to_block::<_, u64, _>(
&FaultyStorage,
(&test_id(1)).into(),
&test_id(100),
).is_err(),
);
// when there's no header in the storage
assert!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new(), &test_id(1), &test_id(100)).is_err());
assert!(
chain::is_connected_to_block::<_, u64, _>(
&DummyStorage::new(),
(&test_id(1)).into(),
&test_id(100),
).is_err(),
);
}
#[test]
@@ -1348,35 +1547,35 @@ pub mod tests {
// when without iterations we end up with different block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(1)),
&test_id(1), &correct_id(1)).unwrap(), false);
(&test_id(1)).into(), &correct_id(1)).unwrap(), false);
// when with ASC iterations we end up with different block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(0))
.with_header(test_header(1))
.with_header(test_header(2)),
&test_id(0), &correct_id(2)).unwrap(), false);
(&test_id(0)).into(), &correct_id(2)).unwrap(), false);
// when with DESC iterations we end up with different block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(0))
.with_header(test_header(1))
.with_header(test_header(2)),
&correct_id(2), &test_id(0)).unwrap(), false);
(&correct_id(2)).into(), &test_id(0)).unwrap(), false);
// when without iterations we end up with the same block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(1)),
&correct_id(1), &correct_id(1)).unwrap(), true);
(&correct_id(1)).into(), &correct_id(1)).unwrap(), true);
// when with ASC iterations we end up with the same block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(0))
.with_header(test_header(1))
.with_header(test_header(2)),
&correct_id(0), &correct_id(2)).unwrap(), true);
(&correct_id(0)).into(), &correct_id(2)).unwrap(), true);
// when with DESC iterations we end up with the same block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(0))
.with_header(test_header(1))
.with_header(test_header(2)),
&correct_id(2), &correct_id(0)).unwrap(), true);
(&correct_id(2)).into(), &correct_id(0)).unwrap(), true);
}
#[test]
@@ -1454,7 +1653,7 @@ pub mod tests {
.with_entry(test_id(10), StorageEntry { prev_valid_from: None, value: 10 })
.with_entry(test_id(20), StorageEntry { prev_valid_from: Some(test_id(10)), value: 20 })
.with_entry(test_id(30), StorageEntry { prev_valid_from: Some(test_id(20)), value: 30 }),
strategy, test_id(9));
strategy, test_id(9)).unwrap();
let mut tx = DummyTransaction::new();
// when finalizing entry #10: no entries pruned
@@ -1515,28 +1714,64 @@ pub mod tests {
.with_header(fork_header(1, 2, 5))
.with_header(fork_header(2, 4, 5)),
PruningStrategy::ByDepth(1024), correct_id(1)
);
).unwrap();
// when 5 is reverted: entry 5 is truncated
let op = cache.on_block_revert(&mut DummyTransaction::new(), &correct_id(5)).unwrap();
let op = cache.do_on_block_revert(&mut DummyTransaction::new(), &correct_id(5)).unwrap();
assert_eq!(op, CommitOperation::BlockReverted(vec![
(0, Some(Fork { best_block: None, head: Entry { valid_from: correct_id(4), value: 4 } })),
].into_iter().collect()));
cache.on_transaction_commit(op);
cache.on_transaction_commit(vec![op].into());
// when 3 is reverted: entries 4+5' are truncated
let op = cache.on_block_revert(&mut DummyTransaction::new(), &correct_id(3)).unwrap();
let op = cache.do_on_block_revert(&mut DummyTransaction::new(), &correct_id(3)).unwrap();
assert_eq!(op, CommitOperation::BlockReverted(vec![
(0, None),
(2, None),
].into_iter().collect()));
cache.on_transaction_commit(op);
cache.on_transaction_commit(vec![op].into());
// when 2 is reverted: entries 4'+5' are truncated
let op = cache.on_block_revert(&mut DummyTransaction::new(), &correct_id(2)).unwrap();
let op = cache.do_on_block_revert(&mut DummyTransaction::new(), &correct_id(2)).unwrap();
assert_eq!(op, CommitOperation::BlockReverted(vec![
(0, None),
].into_iter().collect()));
cache.on_transaction_commit(op);
cache.on_transaction_commit(vec![op].into());
}
#[test]
fn append_commit_operation_works() {
let mut ops = CommitOperations::default();
ops.append(None);
assert_eq!(ops.operations, Vec::new());
ops.append(Some(CommitOperation::BlockFinalized(
test_id(10),
Some(Entry { valid_from: test_id(10), value: 10 }),
vec![5].into_iter().collect(),
)));
assert_eq!(
ops.operations,
vec![CommitOperation::BlockFinalized(
test_id(10),
Some(Entry { valid_from: test_id(10), value: 10 }),
vec![5].into_iter().collect(),
)],
);
ops.append(Some(CommitOperation::BlockFinalized(
test_id(20),
Some(Entry { valid_from: test_id(20), value: 20 }),
vec![5, 6].into_iter().collect(),
)));
assert_eq!(
ops.operations,
vec![CommitOperation::BlockFinalized(
test_id(20),
Some(Entry { valid_from: test_id(20), value: 20 }),
vec![5, 6].into_iter().collect(),
)],
);
}
}
substrate/client/db/src/cache/list_storage.rs
Vendored
+3 -1
View File
@@ -222,7 +222,9 @@ mod meta {
unfinalized.push(&entry.valid_from);
},
CommitOperation::BlockFinalized(_, ref finalizing_entry, ref forks) => {
finalized = finalizing_entry.as_ref().map(|entry| &entry.valid_from);
if let Some(finalizing_entry) = finalizing_entry.as_ref() {
finalized = Some(&finalizing_entry.valid_from);
}
for fork_index in forks.iter().rev() {
unfinalized.remove(*fork_index);
}
substrate/client/db/src/cache/mod.rs
Vendored
+88 -62
View File
@@ -16,7 +16,7 @@
//! DB-backed cache of blockchain data.
use std::{sync::Arc, collections::HashMap};
use std::{sync::Arc, collections::{HashMap, hash_map::Entry}};
use parking_lot::RwLock;
use kvdb::{KeyValueDB, DBTransaction};
@@ -51,8 +51,10 @@ pub enum EntryType {
/// Block identifier that holds both hash and number.
#[derive(Clone, Debug, Encode, Decode, PartialEq)]
pub struct ComplexBlockId<Block: BlockT> {
hash: Block::Hash,
number: NumberFor<Block>,
/// Hash of the block.
pub(crate) hash: Block::Hash,
/// Number of the block.
pub(crate) number: NumberFor<Block>,
}
impl<Block: BlockT> ComplexBlockId<Block> {
@@ -79,7 +81,7 @@ pub struct DbCache<Block: BlockT> {
db: Arc<dyn KeyValueDB>,
key_lookup_column: u32,
header_column: u32,
authorities_column: u32,
cache_column: u32,
genesis_hash: Block::Hash,
best_finalized_block: ComplexBlockId<Block>,
}
@@ -90,7 +92,7 @@ impl<Block: BlockT> DbCache<Block> {
db: Arc<dyn KeyValueDB>,
key_lookup_column: u32,
header_column: u32,
authorities_column: u32,
cache_column: u32,
genesis_hash: Block::Hash,
best_finalized_block: ComplexBlockId<Block>,
) -> Self {
@@ -99,7 +101,7 @@ impl<Block: BlockT> DbCache<Block> {
db,
key_lookup_column,
header_column,
authorities_column,
cache_column,
genesis_hash,
best_finalized_block,
}
@@ -115,30 +117,48 @@ impl<Block: BlockT> DbCache<Block> {
DbCacheTransaction {
cache: self,
tx,
cache_at_op: HashMap::new(),
cache_at_ops: HashMap::new(),
best_finalized_block: None,
}
}
/// Begin cache transaction with given ops.
pub fn transaction_with_ops<'a>(
&'a mut self,
tx: &'a mut DBTransaction,
ops: DbCacheTransactionOps<Block>,
) -> DbCacheTransaction<'a, Block> {
DbCacheTransaction {
cache: self,
tx,
cache_at_ops: ops.cache_at_ops,
best_finalized_block: ops.best_finalized_block,
}
}
/// Run post-commit cache operations.
pub fn commit(&mut self, ops: DbCacheTransactionOps<Block>) {
for (name, op) in ops.cache_at_op.into_iter() {
self.get_cache(name).on_transaction_commit(op);
pub fn commit(&mut self, ops: DbCacheTransactionOps<Block>) -> ClientResult<()> {
for (name, ops) in ops.cache_at_ops.into_iter() {
self.get_cache(name)?.on_transaction_commit(ops);
}
if let Some(best_finalized_block) = ops.best_finalized_block {
self.best_finalized_block = best_finalized_block;
}
Ok(())
}
/// Creates `ListCache` with the given name or returns a reference to the existing.
fn get_cache(&mut self, name: CacheKeyId) -> &mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage> {
pub(crate) fn get_cache(
&mut self,
name: CacheKeyId,
) -> ClientResult<&mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage>> {
get_cache_helper(
&mut self.cache_at,
name,
&self.db,
self.key_lookup_column,
self.header_column,
self.authorities_column,
self.cache_column,
&self.best_finalized_block
)
}
@@ -154,34 +174,49 @@ fn get_cache_helper<'a, Block: BlockT>(
header: u32,
cache: u32,
best_finalized_block: &ComplexBlockId<Block>,
) -> &'a mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage> {
cache_at.entry(name).or_insert_with(|| {
ListCache::new(
self::list_storage::DbStorage::new(name.to_vec(), db.clone(),
self::list_storage::DbColumns {
meta: COLUMN_META,
key_lookup,
header,
cache,
},
),
cache_pruning_strategy(name),
best_finalized_block.clone(),
)
})
) -> ClientResult<&'a mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage>> {
match cache_at.entry(name) {
Entry::Occupied(entry) => Ok(entry.into_mut()),
Entry::Vacant(entry) => {
let cache = ListCache::new(
self::list_storage::DbStorage::new(name.to_vec(), db.clone(),
self::list_storage::DbColumns {
meta: COLUMN_META,
key_lookup,
header,
cache,
},
),
cache_pruning_strategy(name),
best_finalized_block.clone(),
)?;
Ok(entry.insert(cache))
}
}
}
/// Cache operations that are to be committed after database transaction is committed.
#[derive(Default)]
pub struct DbCacheTransactionOps<Block: BlockT> {
cache_at_op: HashMap<CacheKeyId, self::list_cache::CommitOperation<Block, Vec<u8>>>,
cache_at_ops: HashMap<CacheKeyId, self::list_cache::CommitOperations<Block, Vec<u8>>>,
best_finalized_block: Option<ComplexBlockId<Block>>,
}
impl<Block: BlockT> DbCacheTransactionOps<Block> {
/// Empty transaction ops.
pub fn empty() -> DbCacheTransactionOps<Block> {
DbCacheTransactionOps {
cache_at_ops: HashMap::new(),
best_finalized_block: None,
}
}
}
/// Database-backed blockchain data cache transaction valid for single block import.
pub struct DbCacheTransaction<'a, Block: BlockT> {
cache: &'a mut DbCache<Block>,
tx: &'a mut DBTransaction,
cache_at_op: HashMap<CacheKeyId, self::list_cache::CommitOperation<Block, Vec<u8>>>,
cache_at_ops: HashMap<CacheKeyId, self::list_cache::CommitOperations<Block, Vec<u8>>>,
best_finalized_block: Option<ComplexBlockId<Block>>,
}
@@ -189,7 +224,7 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
/// Convert transaction into post-commit operations set.
pub fn into_ops(self) -> DbCacheTransactionOps<Block> {
DbCacheTransactionOps {
cache_at_op: self.cache_at_op,
cache_at_ops: self.cache_at_ops,
best_finalized_block: self.best_finalized_block,
}
}
@@ -202,8 +237,6 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
data_at: HashMap<CacheKeyId, Vec<u8>>,
entry_type: EntryType,
) -> ClientResult<Self> {
assert!(self.cache_at_op.is_empty());
// prepare list of caches that are not update
// (we might still need to do some cache maintenance in this case)
let missed_caches = self.cache.cache_at.keys()
@@ -212,8 +245,9 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
.collect::<Vec<_>>();
let mut insert_op = |name: CacheKeyId, value: Option<Vec<u8>>| -> Result<(), sp_blockchain::Error> {
let cache = self.cache.get_cache(name);
let op = cache.on_block_insert(
let cache = self.cache.get_cache(name)?;
let cache_ops = self.cache_at_ops.entry(name).or_default();
cache.on_block_insert(
&mut self::list_storage::DbStorageTransaction::new(
cache.storage(),
&mut self.tx,
@@ -222,10 +256,9 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
block.clone(),
value,
entry_type,
cache_ops,
)?;
if let Some(op) = op {
self.cache_at_op.insert(name, op);
}
Ok(())
};
@@ -245,23 +278,19 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
pub fn on_block_finalize(
mut self,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>
block: ComplexBlockId<Block>,
) -> ClientResult<Self> {
assert!(self.cache_at_op.is_empty());
for (name, cache_at) in self.cache.cache_at.iter() {
let op = cache_at.on_block_finalize(
for (name, cache) in self.cache.cache_at.iter() {
let cache_ops = self.cache_at_ops.entry(*name).or_default();
cache.on_block_finalize(
&mut self::list_storage::DbStorageTransaction::new(
cache_at.storage(),
cache.storage(),
&mut self.tx
),
parent.clone(),
block.clone(),
cache_ops,
)?;
if let Some(op) = op {
self.cache_at_op.insert(name.to_owned(), op);
}
}
self.best_finalized_block = Some(block);
@@ -275,16 +304,15 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
reverted_block: &ComplexBlockId<Block>,
) -> ClientResult<Self> {
for (name, cache) in self.cache.cache_at.iter() {
let op = cache.on_block_revert(
let cache_ops = self.cache_at_ops.entry(*name).or_default();
cache.on_block_revert(
&mut self::list_storage::DbStorageTransaction::new(
cache.storage(),
&mut self.tx
),
reverted_block,
cache_ops,
)?;
assert!(!self.cache_at_op.contains_key(name));
self.cache_at_op.insert(name.to_owned(), op);
}
Ok(self)
@@ -310,7 +338,7 @@ impl<Block: BlockT> BlockchainCache<Block> for DbCacheSync<Block> {
)?;
let tx_ops = tx.into_ops();
db.write(dbtx).map_err(db_err)?;
cache.commit(tx_ops);
cache.commit(tx_ops)?;
Ok(())
}
@@ -318,40 +346,38 @@ impl<Block: BlockT> BlockchainCache<Block> for DbCacheSync<Block> {
&self,
key: &CacheKeyId,
at: &BlockId<Block>,
) -> Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)> {
) -> ClientResult<Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)>> {
let mut cache = self.0.write();
let storage = cache.get_cache(*key).storage();
let cache = cache.get_cache(*key)?;
let storage = cache.storage();
let db = storage.db();
let columns = storage.columns();
let at = match *at {
BlockId::Hash(hash) => {
let header = utils::read_header::<Block>(
let header = utils::require_header::<Block>(
&**db,
columns.key_lookup,
columns.header,
BlockId::Hash(hash.clone())).ok()??;
BlockId::Hash(hash.clone()))?;
ComplexBlockId::new(hash, *header.number())
},
BlockId::Number(number) => {
let hash = utils::read_header::<Block>(
let hash = utils::require_header::<Block>(
&**db,
columns.key_lookup,
columns.header,
BlockId::Number(number.clone())).ok()??.hash();
BlockId::Number(number.clone()))?.hash();
ComplexBlockId::new(hash, number)
},
};
cache.cache_at
.get(key)?
.value_at_block(&at)
cache.value_at_block(&at)
.map(|block_and_value| block_and_value.map(|(begin_block, end_block, value)|
(
(begin_block.number, begin_block.hash),
end_block.map(|end_block| (end_block.number, end_block.hash)),
value,
)))
.ok()?
}
}
substrate/client/db/src/changes_tries_storage.rs
+1015
View File
File diff suppressed because it is too large Load Diff
substrate/client/db/src/lib.rs
+162 -491
View File
File diff suppressed because it is too large Load Diff
substrate/client/db/src/light.rs
+56 -17
View File
@@ -38,7 +38,7 @@ use codec::{Decode, Encode};
use sp_runtime::generic::{DigestItem, BlockId};
use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, One, NumberFor, HasherFor};
use crate::cache::{DbCacheSync, DbCache, ComplexBlockId, EntryType as CacheEntryType};
use crate::utils::{self, meta_keys, Meta, db_err, read_db, block_id_to_lookup_key, read_meta};
use crate::utils::{self, meta_keys, DatabaseType, Meta, db_err, read_db, block_id_to_lookup_key, read_meta};
use crate::{DatabaseSettings, FrozenForDuration};
use log::{trace, warn, debug};
@@ -68,13 +68,10 @@ pub struct LightStorage<Block: BlockT> {
io_stats: FrozenForDuration<kvdb::IoStats>,
}
impl<Block> LightStorage<Block>
where
Block: BlockT,
{
impl<Block: BlockT> LightStorage<Block> {
/// Create new storage with given settings.
pub fn new(config: DatabaseSettings) -> ClientResult<Self> {
let db = crate::utils::open_database(&config, columns::META, "light")?;
let db = crate::utils::open_database::<Block>(&config, DatabaseType::Light)?;
Self::from_kvdb(db as Arc<_>)
}
@@ -89,7 +86,7 @@ impl<Block> LightStorage<Block>
}
fn from_kvdb(db: Arc<dyn KeyValueDB>) -> ClientResult<Self> {
let meta = read_meta::<Block>(&*db, columns::META, columns::HEADER)?;
let meta = read_meta::<Block>(&*db, columns::HEADER)?;
let cache = DbCache::new(
db.clone(),
columns::KEY_LOOKUP,
@@ -417,7 +414,7 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
fn import_header(
&self,
header: Block::Header,
cache_at: HashMap<well_known_cache_keys::Id, Vec<u8>>,
mut cache_at: HashMap<well_known_cache_keys::Id, Vec<u8>>,
leaf_state: NewBlockState,
aux_ops: Vec<(Vec<u8>, Option<Vec<u8>>)>,
) -> ClientResult<()> {
@@ -475,6 +472,13 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
)?;
}
// update changes trie configuration cache
if !cache_at.contains_key(&well_known_cache_keys::CHANGES_TRIE_CONFIG) {
if let Some(new_configuration) = crate::changes_tries_storage::extract_new_configuration(&header) {
cache_at.insert(well_known_cache_keys::CHANGES_TRIE_CONFIG, new_configuration.encode());
}
}
{
let mut cache = self.cache.0.write();
let cache_ops = cache.transaction(&mut transaction)
@@ -487,8 +491,11 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
.into_ops();
debug!("Light DB Commit {:?} ({})", hash, number);
self.db.write(transaction).map_err(db_err)?;
cache.commit(cache_ops);
cache.commit(cache_ops)
.expect("only fails if cache with given name isn't loaded yet;\
cache is already loaded because there are cache_ops; qed");
}
self.update_meta(hash, number, leaf_state.is_best(), finalized);
@@ -543,7 +550,9 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
.into_ops();
self.db.write(transaction).map_err(db_err)?;
cache.commit(cache_ops);
cache.commit(cache_ops)
.expect("only fails if cache with given name isn't loaded yet;\
cache is already loaded because there are cache_ops; qed");
}
self.update_meta(hash, header.number().clone(), false, true);
@@ -603,7 +612,8 @@ fn cht_key<N: TryInto<u32>>(cht_type: u8, block: N) -> ClientResult<[u8; 5]> {
#[cfg(test)]
pub(crate) mod tests {
use sc_client::cht;
use sp_runtime::generic::DigestItem;
use sp_core::ChangesTrieConfiguration;
use sp_runtime::generic::{DigestItem, ChangesTrieSignal};
use sp_runtime::testing::{H256 as Hash, Header, Block as RawBlock, ExtrinsicWrapper};
use sp_blockchain::{lowest_common_ancestor, tree_route};
use super::*;
@@ -964,7 +974,7 @@ pub(crate) mod tests {
}
fn get_authorities(cache: &dyn BlockchainCache<Block>, at: BlockId<Block>) -> Option<Vec<AuthorityId>> {
cache.get_at(&well_known_cache_keys::AUTHORITIES, &at)
cache.get_at(&well_known_cache_keys::AUTHORITIES, &at).unwrap_or(None)
.and_then(|(_, _, val)| Decode::decode(&mut &val[..]).ok())
}
@@ -1148,8 +1158,8 @@ pub(crate) mod tests {
let (genesis_hash, storage) = {
let db = LightStorage::<Block>::new_test();
// before cache is initialized => None
assert_eq!(db.cache().get_at(b"test", &BlockId::Number(0)), None);
// before cache is initialized => Err
assert!(db.cache().get_at(b"test", &BlockId::Number(0)).is_err());
// insert genesis block (no value for cache is provided)
let mut genesis_hash = None;
@@ -1160,14 +1170,14 @@ pub(crate) mod tests {
});
// after genesis is inserted => None
assert_eq!(db.cache().get_at(b"test", &BlockId::Number(0)), None);
assert_eq!(db.cache().get_at(b"test", &BlockId::Number(0)).unwrap(), None);
// initialize cache
db.cache().initialize(b"test", vec![42]).unwrap();
// after genesis is inserted + cache is initialized => Some
assert_eq!(
db.cache().get_at(b"test", &BlockId::Number(0)),
db.cache().get_at(b"test", &BlockId::Number(0)).unwrap(),
Some(((0, genesis_hash.unwrap()), None, vec![42])),
);
@@ -1177,8 +1187,37 @@ pub(crate) mod tests {
// restart && check that after restart value is read from the cache
let db = LightStorage::<Block>::from_kvdb(storage as Arc<_>).expect("failed to create test-db");
assert_eq!(
db.cache().get_at(b"test", &BlockId::Number(0)),
db.cache().get_at(b"test", &BlockId::Number(0)).unwrap(),
Some(((0, genesis_hash.unwrap()), None, vec![42])),
);
}
#[test]
fn changes_trie_configuration_is_tracked_on_light_client() {
let db = LightStorage::<Block>::new_test();
let new_config = Some(ChangesTrieConfiguration::new(2, 2));
// insert block#0 && block#1 (no value for cache is provided)
let hash0 = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
assert_eq!(
db.cache().get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(0)).unwrap()
.map(|(_, _, v)| ChangesTrieConfiguration::decode(&mut &v[..]).unwrap()),
None,
);
// insert configuration at block#1 (starts from block#2)
insert_block(&db, HashMap::new(), || {
let mut header = default_header(&hash0, 1);
header.digest_mut().push(
DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(new_config.clone()))
);
header
});
assert_eq!(
db.cache().get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(1)).unwrap()
.map(|(_, _, v)| Option::<ChangesTrieConfiguration>::decode(&mut &v[..]).unwrap()),
Some(new_config),
);
}
}
substrate/client/db/src/upgrade.rs
+198
View File
@@ -0,0 +1,198 @@
// Copyright 2019-2020 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Database upgrade logic.
use std::fs;
use std::io::{Read, Write, ErrorKind};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use codec::Encode;
use kvdb_rocksdb::{Database, DatabaseConfig};
use parking_lot::RwLock;
use sp_blockchain::{well_known_cache_keys, Cache};
use sp_core::ChangesTrieConfiguration;
use sp_runtime::traits::Block as BlockT;
use crate::{
cache::{ComplexBlockId, DbCache, DbCacheSync},
utils::{DatabaseType, check_database_type, db_err, read_genesis_hash},
};
/// Version file name.
const VERSION_FILE_NAME: &'static str = "db_version";
/// Current db version.
const CURRENT_VERSION: u32 = 1;
/// Number of columns in v0.
const V0_NUM_COLUMNS: u32 = 10;
/// Upgrade database to current version.
pub fn upgrade_db<Block: BlockT>(db_path: &Path, db_type: DatabaseType) -> sp_blockchain::Result<()> {
let db_version = current_version(db_path)?;
match db_version {
0 => migrate_0_to_1::<Block>(db_path, db_type)?,
1 => (),
_ => Err(sp_blockchain::Error::Backend(format!("Future database version: {}", db_version)))?,
}
update_version(db_path)
}
/// Migration from version0 to version1:
/// 1) the number of columns has changed from 10 to 11;
/// 2) changes tries configuration are now cached.
fn migrate_0_to_1<Block: BlockT>(db_path: &Path, db_type: DatabaseType) -> sp_blockchain::Result<()> {
{
let db = open_database(db_path, db_type, V0_NUM_COLUMNS)?;
db.add_column().map_err(db_err)?;
db.flush().map_err(db_err)?;
}
let db = open_database(db_path, db_type, V0_NUM_COLUMNS + 1)?;
const V0_FULL_KEY_LOOKUP_COLUMN: u32 = 3;
const V0_FULL_HEADER_COLUMN: u32 = 4;
const V0_FULL_CACHE_COLUMN: u32 = 10; // that's the column we have just added
const V0_LIGHT_KEY_LOOKUP_COLUMN: u32 = 1;
const V0_LIGHT_HEADER_COLUMN: u32 = 2;
const V0_LIGHT_CACHE_COLUMN: u32 = 3;
let (key_lookup_column, header_column, cache_column) = match db_type {
DatabaseType::Full => (
V0_FULL_KEY_LOOKUP_COLUMN,
V0_FULL_HEADER_COLUMN,
V0_FULL_CACHE_COLUMN,
),
DatabaseType::Light => (
V0_LIGHT_KEY_LOOKUP_COLUMN,
V0_LIGHT_HEADER_COLUMN,
V0_LIGHT_CACHE_COLUMN,
),
};
let genesis_hash: Option<Block::Hash> = read_genesis_hash(&db)?;
if let Some(genesis_hash) = genesis_hash {
let cache: DbCacheSync<Block> = DbCacheSync(RwLock::new(DbCache::new(
Arc::new(db),
key_lookup_column,
header_column,
cache_column,
genesis_hash,
ComplexBlockId::new(genesis_hash, 0.into()),
)));
let changes_trie_config: Option<ChangesTrieConfiguration> = None;
cache.initialize(&well_known_cache_keys::CHANGES_TRIE_CONFIG, changes_trie_config.encode())?;
}
Ok(())
}
/// Reads current database version from the file at given path.
/// If the file does not exist returns 0.
fn current_version(path: &Path) -> sp_blockchain::Result<u32> {
let unknown_version_err = || sp_blockchain::Error::Backend("Unknown database version".into());
match fs::File::open(version_file_path(path)) {
Err(ref err) if err.kind() == ErrorKind::NotFound => Ok(0),
Err(_) => Err(unknown_version_err()),
Ok(mut file) => {
let mut s = String::new();
file.read_to_string(&mut s).map_err(|_| unknown_version_err())?;
u32::from_str_radix(&s, 10).map_err(|_| unknown_version_err())
},
}
}
/// Opens database of givent type with given number of columns.
fn open_database(db_path: &Path, db_type: DatabaseType, db_columns: u32) -> sp_blockchain::Result<Database> {
let db_path = db_path.to_str()
.ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?;
let db_cfg = DatabaseConfig::with_columns(db_columns);
let db = Database::open(&db_cfg, db_path).map_err(db_err)?;
check_database_type(&db, db_type)?;
Ok(db)
}
/// Writes current database version to the file.
/// Creates a new file if the version file does not exist yet.
fn update_version(path: &Path) -> sp_blockchain::Result<()> {
fs::create_dir_all(path).map_err(db_err)?;
let mut file = fs::File::create(version_file_path(path)).map_err(db_err)?;
file.write_all(format!("{}", CURRENT_VERSION).as_bytes()).map_err(db_err)?;
Ok(())
}
/// Returns the version file path.
fn version_file_path(path: &Path) -> PathBuf {
let mut file_path = path.to_owned();
file_path.push(VERSION_FILE_NAME);
file_path
}
#[cfg(test)]
mod tests {
use sc_state_db::PruningMode;
use crate::{DatabaseSettings, DatabaseSettingsSrc};
use crate::tests::Block;
use super::*;
fn create_db(db_path: &Path, version: Option<u32>) {
let db_cfg = DatabaseConfig::with_columns(V0_NUM_COLUMNS);
Database::open(&db_cfg, db_path.to_str().unwrap()).unwrap();
if let Some(version) = version {
fs::create_dir_all(db_path).unwrap();
let mut file = fs::File::create(version_file_path(db_path)).unwrap();
file.write_all(format!("{}", version).as_bytes()).unwrap();
}
}
fn open_database(db_path: &Path) -> sp_blockchain::Result<()> {
crate::utils::open_database::<Block>(&DatabaseSettings {
state_cache_size: 0,
state_cache_child_ratio: None,
pruning: PruningMode::ArchiveAll,
source: DatabaseSettingsSrc::Path { path: db_path.to_owned(), cache_size: None },
}, DatabaseType::Full).map(|_| ())
}
#[test]
fn downgrade_never_happens() {
let db_dir = tempdir::TempDir::new("").unwrap();
create_db(db_dir.path(), Some(CURRENT_VERSION + 1));
assert!(open_database(db_dir.path()).is_err());
}
#[test]
fn open_empty_database_works() {
let db_dir = tempdir::TempDir::new("").unwrap();
open_database(db_dir.path()).unwrap();
open_database(db_dir.path()).unwrap();
assert_eq!(current_version(db_dir.path()).unwrap(), CURRENT_VERSION);
}
#[test]
fn upgrade_from_0_to_1_works() {
for version_from_file in &[None, Some(0)] {
let db_dir = tempdir::TempDir::new("").unwrap();
let db_path = db_dir.path();
create_db(db_path, *version_from_file);
open_database(db_path).unwrap();
assert_eq!(current_version(db_path).unwrap(), CURRENT_VERSION);
}
}
}
substrate/client/db/src/utils.rs
+66 -22
View File
@@ -21,7 +21,7 @@ use std::sync::Arc;
use std::{io, convert::TryInto};
use kvdb::{KeyValueDB, DBTransaction};
#[cfg(feature = "kvdb-rocksdb")]
#[cfg(any(feature = "kvdb-rocksdb", test))]
use kvdb_rocksdb::{Database, DatabaseConfig};
use log::debug;
@@ -36,7 +36,7 @@ use crate::{DatabaseSettings, DatabaseSettingsSrc};
/// Number of columns in the db. Must be the same for both full && light dbs.
/// Otherwise RocksDb will fail to open database && check its type.
pub const NUM_COLUMNS: u32 = 10;
pub const NUM_COLUMNS: u32 = 11;
/// Meta column. The set of keys in the column is shared by full && light storages.
pub const COLUMN_META: u32 = 0;
@@ -50,6 +50,8 @@ pub mod meta_keys {
pub const FINALIZED_BLOCK: &[u8; 5] = b"final";
/// Meta information prefix for list-based caches.
pub const CACHE_META_PREFIX: &[u8; 5] = b"cache";
/// Meta information for changes tries key.
pub const CHANGES_TRIES_META: &[u8; 5] = b"ctrie";
/// Genesis block hash.
pub const GENESIS_HASH: &[u8; 3] = b"gen";
/// Leaves prefix list key.
@@ -76,6 +78,15 @@ pub struct Meta<N, H> {
/// A block lookup key: used for canonical lookup from block number to hash
pub type NumberIndexKey = [u8; 4];
/// Database type.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum DatabaseType {
/// Full node database.
Full,
/// Light node database.
Light,
}
/// Convert block number into short lookup key (LE representation) for
/// blocks that are in the canonical chain.
///
@@ -203,14 +214,17 @@ pub fn db_err(err: io::Error) -> sp_blockchain::Error {
}
/// Open RocksDB database.
pub fn open_database(
pub fn open_database<Block: BlockT>(
config: &DatabaseSettings,
col_meta: u32,
db_type: &str
db_type: DatabaseType,
) -> sp_blockchain::Result<Arc<dyn KeyValueDB>> {
let db: Arc<dyn KeyValueDB> = match &config.source {
#[cfg(feature = "kvdb-rocksdb")]
#[cfg(any(feature = "kvdb-rocksdb", test))]
DatabaseSettingsSrc::Path { path, cache_size } => {
// first upgrade database to required version
crate::upgrade::upgrade_db::<Block>(&path, db_type)?;
// and now open database assuming that it has the latest version
let mut db_config = DatabaseConfig::with_columns(NUM_COLUMNS);
if let Some(cache_size) = cache_size {
@@ -232,7 +246,7 @@ pub fn open_database(
.ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?;
Arc::new(Database::open(&db_config, &path).map_err(db_err)?)
},
#[cfg(not(feature = "kvdb-rocksdb"))]
#[cfg(not(any(feature = "kvdb-rocksdb", test)))]
DatabaseSettingsSrc::Path { .. } => {
let msg = "Try to open RocksDB database with RocksDB disabled".into();
return Err(sp_blockchain::Error::Backend(msg));
@@ -240,22 +254,28 @@ pub fn open_database(
DatabaseSettingsSrc::Custom(db) => db.clone(),
};
// check database type
match db.get(col_meta, meta_keys::TYPE).map_err(db_err)? {
check_database_type(&*db, db_type)?;
Ok(db)
}
/// Check database type.
pub fn check_database_type(db: &dyn KeyValueDB, db_type: DatabaseType) -> sp_blockchain::Result<()> {
match db.get(COLUMN_META, meta_keys::TYPE).map_err(db_err)? {
Some(stored_type) => {
if db_type.as_bytes() != &*stored_type {
if db_type.as_str().as_bytes() != &*stored_type {
return Err(sp_blockchain::Error::Backend(
format!("Unexpected database type. Expected: {}", db_type)).into());
format!("Unexpected database type. Expected: {}", db_type.as_str())).into());
}
},
None => {
let mut transaction = DBTransaction::new();
transaction.put(col_meta, meta_keys::TYPE, db_type.as_bytes());
transaction.put(COLUMN_META, meta_keys::TYPE, db_type.as_str().as_bytes());
db.write(transaction).map_err(db_err)?;
},
}
Ok(db)
Ok(())
}
/// Read database column entry for the given block.
@@ -304,20 +324,15 @@ pub fn require_header<Block: BlockT>(
}
/// Read meta from the database.
pub fn read_meta<Block>(db: &dyn KeyValueDB, col_meta: u32, col_header: u32) -> Result<
pub fn read_meta<Block>(db: &dyn KeyValueDB, col_header: u32) -> Result<
Meta<<<Block as BlockT>::Header as HeaderT>::Number, Block::Hash>,
sp_blockchain::Error,
>
where
Block: BlockT,
{
let genesis_hash: Block::Hash = match db.get(col_meta, meta_keys::GENESIS_HASH).map_err(db_err)? {
Some(h) => match Decode::decode(&mut &h[..]) {
Ok(h) => h,
Err(err) => return Err(sp_blockchain::Error::Backend(
format!("Error decoding genesis hash: {}", err)
)),
},
let genesis_hash: Block::Hash = match read_genesis_hash(db)? {
Some(genesis_hash) => genesis_hash,
None => return Ok(Meta {
best_hash: Default::default(),
best_number: Zero::zero(),
@@ -328,7 +343,7 @@ pub fn read_meta<Block>(db: &dyn KeyValueDB, col_meta: u32, col_header: u32) ->
};
let load_meta_block = |desc, key| -> Result<_, sp_blockchain::Error> {
if let Some(Some(header)) = db.get(col_meta, key).and_then(|id|
if let Some(Some(header)) = db.get(COLUMN_META, key).and_then(|id|
match id {
Some(id) => db.get(col_header, &id).map(|h| h.map(|b| Block::Header::decode(&mut &b[..]).ok())),
None => Ok(None),
@@ -354,6 +369,29 @@ pub fn read_meta<Block>(db: &dyn KeyValueDB, col_meta: u32, col_header: u32) ->
})
}
/// Read genesis hash from database.
pub fn read_genesis_hash<Hash: Decode>(db: &dyn KeyValueDB) -> sp_blockchain::Result<Option<Hash>> {
match db.get(COLUMN_META, meta_keys::GENESIS_HASH).map_err(db_err)? {
Some(h) => match Decode::decode(&mut &h[..]) {
Ok(h) => Ok(Some(h)),
Err(err) => Err(sp_blockchain::Error::Backend(
format!("Error decoding genesis hash: {}", err)
)),
},
None => Ok(None),
}
}
impl DatabaseType {
/// Returns str representation of the type.
pub fn as_str(&self) -> &'static str {
match *self {
DatabaseType::Full => "full",
DatabaseType::Light => "light",
}
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -368,4 +406,10 @@ mod tests {
_ => unreachable!(),
};
}
#[test]
fn database_type_as_str_works() {
assert_eq!(DatabaseType::Full.as_str(), "full");
assert_eq!(DatabaseType::Light.as_str(), "light");
}
}
substrate/client/finality-grandpa/src/tests.rs
+2 -4
View File
@@ -287,12 +287,10 @@ impl ApiExt<Block> for RuntimeApi {
unimplemented!("Not required for testing!")
}
fn into_storage_changes<
T: sp_api::ChangesTrieStorage<sp_api::HasherFor<Block>, sp_api::NumberFor<Block>>
>(
fn into_storage_changes(
&self,
_: &Self::StateBackend,
_: Option<&T>,
_: Option<&sp_api::ChangesTrieState<sp_api::HasherFor<Block>, sp_api::NumberFor<Block>>>,
_: <Block as sp_api::BlockT>::Hash,
) -> std::result::Result<sp_api::StorageChanges<Self::StateBackend, Block>, String>
where Self: Sized
substrate/client/network/src/protocol/light_dispatch.rs
+6 -2
View File
@@ -691,7 +691,7 @@ pub mod tests {
use crate::message::{self, BlockAttributes, Direction, FromBlock, RequestId};
use libp2p::PeerId;
use super::{REQUEST_TIMEOUT, LightDispatch, LightDispatchNetwork, RequestData, StorageProof};
use sp_test_primitives::{changes_trie_config, Block, Extrinsic, Header};
use sp_test_primitives::{Block, Extrinsic, Header};
struct DummyFetchChecker { ok: bool }
@@ -1095,7 +1095,11 @@ pub mod tests {
let (tx, response) = oneshot::channel();
light_dispatch.add_request(&mut network_interface, RequestData::RemoteChanges(RemoteChangesRequest {
changes_trie_config: changes_trie_config(),
changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange {
zero: (0, Default::default()),
end: None,
config: Some(sp_core::ChangesTrieConfiguration::new(4, 2)),
}],
first_block: (1, Default::default()),
last_block: (100, Default::default()),
max_block: (100, Default::default()),
substrate/client/rpc/src/state/tests.rs
+4 -2
View File
@@ -21,7 +21,7 @@ use self::error::Error;
use std::sync::Arc;
use assert_matches::assert_matches;
use futures01::stream::Stream;
use sp_core::storage::{well_known_keys, ChildInfo};
use sp_core::{storage::{well_known_keys, ChildInfo}, ChangesTrieConfiguration};
use sp_core::hash::H256;
use sp_io::hashing::blake2_256;
use substrate_test_runtime_client::{
@@ -378,7 +378,9 @@ fn should_query_storage() {
}
run_tests(Arc::new(substrate_test_runtime_client::new()));
run_tests(Arc::new(TestClientBuilder::new().set_support_changes_trie(true).build()));
run_tests(Arc::new(TestClientBuilder::new()
.changes_trie_config(Some(ChangesTrieConfiguration::new(4, 2)))
.build()));
}
#[test]
substrate/client/src/call_executor.rs
+8 -7
View File
@@ -17,7 +17,7 @@
use std::{sync::Arc, panic::UnwindSafe, result, cell::RefCell};
use codec::{Encode, Decode};
use sp_runtime::{
generic::BlockId, traits::{Block as BlockT, HasherFor},
generic::BlockId, traits::{Block as BlockT, HasherFor, NumberFor},
};
use sp_state_machine::{
self, OverlayedChanges, Ext, ExecutionManager, StateMachine, ExecutionStrategy,
@@ -80,7 +80,7 @@ where
let state = self.backend.state_at(*id)?;
let return_data = StateMachine::new(
&state,
self.backend.changes_trie_storage(),
backend::changes_tries_state_at_block(id, self.backend.changes_trie_storage())?,
&mut changes,
&self.executor,
method,
@@ -132,6 +132,7 @@ where
}
let mut state = self.backend.state_at(*at)?;
let changes_trie_state = backend::changes_tries_state_at_block(at, self.backend.changes_trie_storage())?;
let mut storage_transaction_cache = storage_transaction_cache.map(|c| c.borrow_mut());
@@ -150,7 +151,7 @@ where
StateMachine::new(
&backend,
self.backend.changes_trie_storage(),
changes_trie_state,
&mut *changes.borrow_mut(),
&self.executor,
method,
@@ -163,7 +164,7 @@ where
}
None => StateMachine::new(
&state,
self.backend.changes_trie_storage(),
changes_trie_state,
&mut *changes.borrow_mut(),
&self.executor,
method,
@@ -183,13 +184,13 @@ where
fn runtime_version(&self, id: &BlockId<Block>) -> sp_blockchain::Result<RuntimeVersion> {
let mut overlay = OverlayedChanges::default();
let state = self.backend.state_at(*id)?;
let changes_trie_state = backend::changes_tries_state_at_block(id, self.backend.changes_trie_storage())?;
let mut cache = StorageTransactionCache::<Block, B::State>::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
&state,
self.backend.changes_trie_storage(),
changes_trie_state,
None,
);
let version = self.executor.runtime_version(&mut ext);
@@ -207,7 +208,7 @@ where
method: &str,
call_data: &[u8]
) -> Result<(Vec<u8>, StorageProof), sp_blockchain::Error> {
sp_state_machine::prove_execution_on_trie_backend(
sp_state_machine::prove_execution_on_trie_backend::<_, _, NumberFor<Block>, _>(
trie_state,
overlay,
&self.executor,
substrate/client/src/client.rs
+230 -76
View File
@@ -66,6 +66,7 @@ pub use sc_client_api::{
backend::{
self, BlockImportOperation, PrunableStateChangesTrieStorage,
ClientImportOperation, Finalizer, ImportSummary, NewBlockState,
changes_tries_state_at_block,
},
client::{
ImportNotifications, FinalityNotification, FinalityNotifications, BlockImportNotification,
@@ -407,18 +408,26 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
first: NumberFor<Block>,
last: BlockId<Block>,
) -> sp_blockchain::Result<Option<(NumberFor<Block>, BlockId<Block>)>> {
let (config, storage) = match self.require_changes_trie().ok() {
Some((config, storage)) => (config, storage),
None => return Ok(None),
};
let last_num = self.backend.blockchain().expect_block_number_from_id(&last)?;
if first > last_num {
let last_number = self.backend.blockchain().expect_block_number_from_id(&last)?;
let last_hash = self.backend.blockchain().expect_block_hash_from_id(&last)?;
if first > last_number {
return Err(sp_blockchain::Error::ChangesTrieAccessFailed("Invalid changes trie range".into()));
}
let finalized_number = self.backend.blockchain().info().finalized_number;
let oldest = storage.oldest_changes_trie_block(&config, finalized_number);
let first = ::std::cmp::max(first, oldest);
Ok(Some((first, last)))
let (storage, configs) = match self.require_changes_trie(first, last_hash, false).ok() {
Some((storage, configs)) => (storage, configs),
None => return Ok(None),
};
let first_available_changes_trie = configs.last().map(|config| config.0);
match first_available_changes_trie {
Some(first_available_changes_trie) => {
let oldest_unpruned = storage.oldest_pruned_digest_range_end();
let first = std::cmp::max(first_available_changes_trie, oldest_unpruned);
Ok(Some((first, last)))
},
None => Ok(None)
}
}
/// Get pairs of (block, extrinsic) where key has been changed at given blocks range.
@@ -432,30 +441,42 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
storage_key: Option<&StorageKey>,
key: &StorageKey
) -> sp_blockchain::Result<Vec<(NumberFor<Block>, u32)>> {
let (config, storage) = self.require_changes_trie()?;
let last_number = self.backend.blockchain().expect_block_number_from_id(&last)?;
let last_hash = self.backend.blockchain().expect_block_hash_from_id(&last)?;
let (storage, configs) = self.require_changes_trie(first, last_hash, true)?;
// FIXME: remove this in https://github.com/paritytech/substrate/pull/3201
let config_range = ChangesTrieConfigurationRange {
config: &config,
zero: Zero::zero(),
end: None,
};
let mut result = Vec::new();
let best_number = self.backend.blockchain().info().best_number;
for (config_zero, config_end, config) in configs {
let range_first = ::std::cmp::max(first, config_zero + One::one());
let range_anchor = match config_end {
Some((config_end_number, config_end_hash)) => if last_number > config_end_number {
ChangesTrieAnchorBlockId { hash: config_end_hash, number: config_end_number }
} else {
ChangesTrieAnchorBlockId { hash: convert_hash(&last_hash), number: last_number }
},
None => ChangesTrieAnchorBlockId { hash: convert_hash(&last_hash), number: last_number },
};
key_changes::<HasherFor<Block>, _>(
config_range,
&*storage,
first,
&ChangesTrieAnchorBlockId {
hash: convert_hash(&last_hash),
number: last_number,
},
self.backend.blockchain().info().best_number,
storage_key.as_ref().map(|sk| sk.0.as_slice()),
&key.0)
.and_then(|r| r.map(|r| r.map(|(block, tx)| (block, tx))).collect::<Result<_, _>>())
.map_err(|err| sp_blockchain::Error::ChangesTrieAccessFailed(err))
let config_range = ChangesTrieConfigurationRange {
config: &config,
zero: config_zero.clone(),
end: config_end.map(|(config_end_number, _)| config_end_number),
};
let result_range: Vec<(NumberFor<Block>, u32)> = key_changes::<HasherFor<Block>, _>(
config_range,
storage.storage(),
range_first,
&range_anchor,
best_number,
storage_key.as_ref().map(|x| &x.0[..]),
&key.0)
.and_then(|r| r.map(|r| r.map(|(block, tx)| (block, tx))).collect::<Result<_, _>>())
.map_err(|err| sp_blockchain::Error::ChangesTrieAccessFailed(err))?;
result.extend(result_range);
}
Ok(result)
}
/// Get proof for computation of (block, extrinsic) pairs where key has been changed at given blocks range.
@@ -550,11 +571,13 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
}
}
let (config, storage) = self.require_changes_trie()?;
let first_number = self.backend.blockchain()
.expect_block_number_from_id(&BlockId::Hash(first))?;
let (storage, configs) = self.require_changes_trie(first_number, last, true)?;
let min_number = self.backend.blockchain().expect_block_number_from_id(&BlockId::Hash(min))?;
let recording_storage = AccessedRootsRecorder::<Block> {
storage,
storage: storage.storage(),
min: min_number,
required_roots_proofs: Mutex::new(BTreeMap::new()),
};
@@ -564,31 +587,31 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
self.backend.blockchain().expect_block_number_from_id(&BlockId::Hash(max))?,
);
// FIXME: remove this in https://github.com/paritytech/substrate/pull/3201
let config_range = ChangesTrieConfigurationRange {
config: &config,
zero: Zero::zero(),
end: None,
};
// fetch key changes proof
let first_number = self.backend.blockchain()
.expect_block_number_from_id(&BlockId::Hash(first))?;
let last_number = self.backend.blockchain()
.expect_block_number_from_id(&BlockId::Hash(last))?;
let key_changes_proof = key_changes_proof::<HasherFor<Block>, _>(
config_range,
&recording_storage,
first_number,
&ChangesTrieAnchorBlockId {
hash: convert_hash(&last),
number: last_number,
},
max_number,
storage_key.as_ref().map(|sk| sk.0.as_slice()),
&key.0,
)
.map_err(|err| sp_blockchain::Error::from(sp_blockchain::Error::ChangesTrieAccessFailed(err)))?;
let mut proof = Vec::new();
for (config_zero, config_end, config) in configs {
let last_number = self.backend.blockchain()
.expect_block_number_from_id(&BlockId::Hash(last))?;
let config_range = ChangesTrieConfigurationRange {
config: &config,
zero: config_zero,
end: config_end.map(|(config_end_number, _)| config_end_number),
};
let proof_range = key_changes_proof::<HasherFor<Block>, _>(
config_range,
&recording_storage,
first_number,
&ChangesTrieAnchorBlockId {
hash: convert_hash(&last),
number: last_number,
},
max_number,
storage_key.as_ref().map(|x| &x.0[..]),
&key.0,
)
.map_err(|err| sp_blockchain::Error::ChangesTrieAccessFailed(err))?;
proof.extend(proof_range);
}
// now gather proofs for all changes tries roots that were touched during key_changes_proof
// execution AND are unknown (i.e. replaced with CHT) to the requester
@@ -597,7 +620,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
Ok(ChangesProof {
max_block: max_number,
proof: key_changes_proof,
proof,
roots: roots.into_iter().map(|(n, h)| (n, convert_hash(&h))).collect(),
roots_proof,
})
@@ -650,14 +673,45 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
Ok(proof)
}
/// Returns changes trie configuration and storage or an error if it is not supported.
fn require_changes_trie(&self) -> sp_blockchain::Result<(ChangesTrieConfiguration, &B::ChangesTrieStorage)> {
let config = self.changes_trie_config()?;
let storage = self.backend.changes_trie_storage();
match (config, storage) {
(Some(config), Some(storage)) => Ok((config, storage)),
_ => Err(sp_blockchain::Error::ChangesTriesNotSupported.into()),
/// Returns changes trie storage and all configurations that have been active in the range [first; last].
///
/// Configurations are returned in descending order (and obviously never overlap).
/// If fail_if_disabled is false, returns maximal consequent configurations ranges, starting from last and
/// stopping on either first, or when CT have been disabled.
/// If fail_if_disabled is true, fails when there's a subrange where CT have been disabled
/// inside first..last blocks range.
fn require_changes_trie(
&self,
first: NumberFor<Block>,
last: Block::Hash,
fail_if_disabled: bool,
) -> sp_blockchain::Result<(
&dyn PrunableStateChangesTrieStorage<Block>,
Vec<(NumberFor<Block>, Option<(NumberFor<Block>, Block::Hash)>, ChangesTrieConfiguration)>,
)> {
let storage = match self.backend.changes_trie_storage() {
Some(storage) => storage,
None => return Err(sp_blockchain::Error::ChangesTriesNotSupported),
};
let mut configs = Vec::with_capacity(1);
let mut current = last;
loop {
let config_range = storage.configuration_at(&BlockId::Hash(current))?;
match config_range.config {
Some(config) => configs.push((config_range.zero.0, config_range.end, config)),
None if !fail_if_disabled => return Ok((storage, configs)),
None => return Err(sp_blockchain::Error::ChangesTriesNotSupported),
}
if config_range.zero.0 < first {
break;
}
current = *self.backend.blockchain().expect_header(BlockId::Hash(config_range.zero.1))?.parent_hash();
}
Ok((storage, configs))
}
/// Create a new block, built on the head of the chain.
@@ -988,10 +1042,14 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
)?;
let state = self.backend.state_at(at)?;
let changes_trie_state = changes_tries_state_at_block(
&at,
self.backend.changes_trie_storage(),
)?;
let gen_storage_changes = runtime_api.into_storage_changes(
&state,
self.backend.changes_trie_storage(),
changes_trie_state.as_ref(),
*parent_hash,
);
@@ -1249,13 +1307,6 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
Ok(uncles)
}
fn changes_trie_config(&self) -> Result<Option<ChangesTrieConfiguration>, Error> {
Ok(self.backend.state_at(BlockId::Number(self.backend.blockchain().info().best_number))?
.storage(well_known_keys::CHANGES_TRIE_CONFIG)
.map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))?
.and_then(|c| Decode::decode(&mut &*c).ok()))
}
/// Prepare in-memory header that is used in execution environment.
fn prepare_environment_block(&self, parent: &BlockId<Block>) -> sp_blockchain::Result<Block::Header> {
let parent_header = self.backend.blockchain().expect_header(*parent)?;
@@ -1871,7 +1922,8 @@ pub(crate) mod tests {
// prepare client ang import blocks
let mut local_roots = Vec::new();
let mut remote_client = TestClientBuilder::new().set_support_changes_trie(true).build();
let config = Some(ChangesTrieConfiguration::new(4, 2));
let mut remote_client = TestClientBuilder::new().changes_trie_config(config).build();
let mut nonces: HashMap<_, u64> = Default::default();
for (i, block_transfers) in blocks_transfers.into_iter().enumerate() {
let mut builder = remote_client.new_block(Default::default()).unwrap();
@@ -2907,13 +2959,15 @@ pub(crate) mod tests {
.unwrap().build().unwrap().block;
client.import(BlockOrigin::Own, b2.clone()).unwrap();
// prepare B3 before we finalize A2, because otherwise we won't be able to
// read changes trie configuration after A2 is finalized
let b3 = client.new_block_at(&BlockId::Hash(b2.hash()), Default::default(), false)
.unwrap().build().unwrap().block;
// we will finalize A2 which should make it impossible to import a new
// B3 at the same height but that doesnt't include it
ClientExt::finalize_block(&client, BlockId::Hash(a2.hash()), None).unwrap();
let b3 = client.new_block_at(&BlockId::Hash(b2.hash()), Default::default(), false)
.unwrap().build().unwrap().block;
let import_err = client.import(BlockOrigin::Own, b3).err().unwrap();
let expected_err = ConsensusError::ClientImport(
sp_blockchain::Error::NotInFinalizedChain.to_string()
@@ -3050,4 +3104,104 @@ pub(crate) mod tests {
check_block_b1.parent_hash = H256::random();
assert_eq!(client.check_block(check_block_b1.clone()).unwrap(), ImportResult::UnknownParent);
}
#[test]
fn imports_blocks_with_changes_tries_config_change() {
// create client with initial 4^2 configuration
let mut client = TestClientBuilder::with_default_backend()
.changes_trie_config(Some(ChangesTrieConfiguration {
digest_interval: 4,
digest_levels: 2,
})).build();
// ===================================================================
// blocks 1,2,3,4,5,6,7,8,9,10 are empty
// block 11 changes the key
// block 12 is the L1 digest that covers this change
// blocks 13,14,15,16,17,18,19,20,21,22 are empty
// block 23 changes the configuration to 5^1 AND is skewed digest
// ===================================================================
// blocks 24,25 are changing the key
// block 26 is empty
// block 27 changes the key
// block 28 is the L1 digest (NOT SKEWED!!!) that covers changes AND changes configuration to 3^1
// ===================================================================
// block 29 is empty
// block 30 changes the key
// block 31 is L1 digest that covers this change
// ===================================================================
(1..11).for_each(|number| {
let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false)
.unwrap().build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(11..12).for_each(|number| {
let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap();
block.push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())).unwrap();
let block = block.build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(12..23).for_each(|number| {
let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false)
.unwrap().build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(23..24).for_each(|number| {
let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap();
block.push_changes_trie_configuration_update(Some(ChangesTrieConfiguration {
digest_interval: 5,
digest_levels: 1,
})).unwrap();
let block = block.build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(24..26).for_each(|number| {
let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap();
block.push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())).unwrap();
let block = block.build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(26..27).for_each(|number| {
let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false)
.unwrap().build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(27..28).for_each(|number| {
let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap();
block.push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())).unwrap();
let block = block.build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(28..29).for_each(|number| {
let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap();
block.push_changes_trie_configuration_update(Some(ChangesTrieConfiguration {
digest_interval: 3,
digest_levels: 1,
})).unwrap();
let block = block.build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(29..30).for_each(|number| {
let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false)
.unwrap().build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(30..31).for_each(|number| {
let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap();
block.push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())).unwrap();
let block = block.build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(31..32).for_each(|number| {
let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false)
.unwrap().build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
// now check that configuration cache works
assert_eq!(
client.key_changes(1, BlockId::Number(31), None, &StorageKey(vec![42])).unwrap(),
vec![(30, 0), (27, 0), (25, 0), (24, 0), (11, 0)]
);
}
}
substrate/client/src/genesis.rs
+10 -10
View File
@@ -45,7 +45,7 @@ mod tests {
use codec::{Encode, Decode, Joiner};
use sc_executor::native_executor_instance;
use sp_state_machine::{
StateMachine, OverlayedChanges, ExecutionStrategy, InMemoryChangesTrieStorage,
StateMachine, OverlayedChanges, ExecutionStrategy,
InMemoryBackend,
};
use substrate_test_runtime_client::{
@@ -92,7 +92,7 @@ mod tests {
StateMachine::new(
backend,
Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
sp_state_machine::disabled_changes_trie_state::<_, u64>(),
&mut overlay,
&executor(),
"Core_initialize_block",
@@ -105,7 +105,7 @@ mod tests {
for tx in transactions.iter() {
StateMachine::new(
backend,
Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
sp_state_machine::disabled_changes_trie_state::<_, u64>(),
&mut overlay,
&executor(),
"BlockBuilder_apply_extrinsic",
@@ -118,7 +118,7 @@ mod tests {
let ret_data = StateMachine::new(
backend,
Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
sp_state_machine::disabled_changes_trie_state::<_, u64>(),
&mut overlay,
&executor(),
"BlockBuilder_finalize_block",
@@ -150,7 +150,7 @@ mod tests {
#[test]
fn construct_genesis_should_work_with_native() {
let mut storage = GenesisConfig::new(
false,
None,
vec![Sr25519Keyring::One.public().into(), Sr25519Keyring::Two.public().into()],
vec![AccountKeyring::One.into(), AccountKeyring::Two.into()],
1000,
@@ -165,7 +165,7 @@ mod tests {
let mut overlay = OverlayedChanges::default();
let _ = StateMachine::new(
&backend,
Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
sp_state_machine::disabled_changes_trie_state::<_, u64>(),
&mut overlay,
&executor(),
"Core_execute_block",
@@ -178,7 +178,7 @@ mod tests {
#[test]
fn construct_genesis_should_work_with_wasm() {
let mut storage = GenesisConfig::new(false,
let mut storage = GenesisConfig::new(None,
vec![Sr25519Keyring::One.public().into(), Sr25519Keyring::Two.public().into()],
vec![AccountKeyring::One.into(), AccountKeyring::Two.into()],
1000,
@@ -193,7 +193,7 @@ mod tests {
let mut overlay = OverlayedChanges::default();
let _ = StateMachine::new(
&backend,
Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
sp_state_machine::disabled_changes_trie_state::<_, u64>(),
&mut overlay,
&executor(),
"Core_execute_block",
@@ -206,7 +206,7 @@ mod tests {
#[test]
fn construct_genesis_with_bad_transaction_should_panic() {
let mut storage = GenesisConfig::new(false,
let mut storage = GenesisConfig::new(None,
vec![Sr25519Keyring::One.public().into(), Sr25519Keyring::Two.public().into()],
vec![AccountKeyring::One.into(), AccountKeyring::Two.into()],
68,
@@ -221,7 +221,7 @@ mod tests {
let mut overlay = OverlayedChanges::default();
let r = StateMachine::new(
&backend,
Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
sp_state_machine::disabled_changes_trie_state::<_, u64>(),
&mut overlay,
&executor(),
"Core_execute_block",
substrate/client/src/in_mem.rs
+7 -89
View File
@@ -16,22 +16,17 @@
//! In memory client backend
use std::collections::{HashMap, HashSet};
use std::collections::HashMap;
use std::sync::Arc;
use parking_lot::RwLock;
use sp_core::{ChangesTrieConfiguration, storage::well_known_keys};
use sp_core::storage::well_known_keys;
use sp_core::offchain::storage::{
InMemOffchainStorage as OffchainStorage
};
use sp_runtime::generic::{BlockId, DigestItem};
use sp_runtime::generic::BlockId;
use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, NumberFor, HasherFor};
use sp_runtime::{Justification, Storage};
use sp_state_machine::{
InMemoryChangesTrieStorage, ChangesTrieAnchorBlockId, ChangesTrieTransaction,
InMemoryBackend, Backend as StateBackend,
};
use hash_db::Prefix;
use sp_trie::MemoryDB;
use sp_state_machine::{ChangesTrieTransaction, InMemoryBackend, Backend as StateBackend};
use sp_blockchain::{CachedHeaderMetadata, HeaderMetadata};
use sc_client_api::{
@@ -466,7 +461,6 @@ pub struct BlockImportOperation<Block: BlockT> {
pending_cache: HashMap<CacheKeyId, Vec<u8>>,
old_state: InMemoryBackend<HasherFor<Block>>,
new_state: Option<InMemoryBackend<HasherFor<Block>>>,
changes_trie_update: Option<MemoryDB<HasherFor<Block>>>,
aux: Vec<(Vec<u8>, Option<Vec<u8>>)>,
finalized_blocks: Vec<(BlockId<Block>, Option<Justification>)>,
set_head: Option<BlockId<Block>>,
@@ -510,9 +504,8 @@ impl<Block: BlockT> backend::BlockImportOperation<Block> for BlockImportOperatio
fn update_changes_trie(
&mut self,
update: ChangesTrieTransaction<HasherFor<Block>, NumberFor<Block>>,
_update: ChangesTrieTransaction<HasherFor<Block>, NumberFor<Block>>,
) -> sp_blockchain::Result<()> {
self.changes_trie_update = Some(update.0);
Ok(())
}
@@ -569,7 +562,6 @@ impl<Block: BlockT> backend::BlockImportOperation<Block> for BlockImportOperatio
/// > struct for testing purposes. Do **NOT** use in production.
pub struct Backend<Block: BlockT> where Block::Hash: Ord {
states: RwLock<HashMap<Block::Hash, InMemoryBackend<HasherFor<Block>>>>,
changes_trie_storage: ChangesTrieStorage<Block>,
blockchain: Blockchain<Block>,
import_lock: RwLock<()>,
}
@@ -579,7 +571,6 @@ impl<Block: BlockT> Backend<Block> where Block::Hash: Ord {
pub fn new() -> Self {
Backend {
states: RwLock::new(HashMap::new()),
changes_trie_storage: ChangesTrieStorage(InMemoryChangesTrieStorage::new()),
blockchain: Blockchain::new(),
import_lock: Default::default(),
}
@@ -606,7 +597,6 @@ impl<Block: BlockT> backend::Backend<Block> for Backend<Block> where Block::Hash
type BlockImportOperation = BlockImportOperation<Block>;
type Blockchain = Blockchain<Block>;
type State = InMemoryBackend<HasherFor<Block>>;
type ChangesTrieStorage = ChangesTrieStorage<Block>;
type OffchainStorage = OffchainStorage;
fn begin_operation(&self) -> sp_blockchain::Result<Self::BlockImportOperation> {
@@ -616,7 +606,6 @@ impl<Block: BlockT> backend::Backend<Block> for Backend<Block> where Block::Hash
pending_cache: Default::default(),
old_state,
new_state: None,
changes_trie_update: None,
aux: Default::default(),
finalized_blocks: Default::default(),
set_head: None,
@@ -647,17 +636,6 @@ impl<Block: BlockT> backend::Backend<Block> for Backend<Block> where Block::Hash
self.states.write().insert(hash, operation.new_state.unwrap_or_else(|| old_state.clone()));
let maybe_changes_trie_root = header.digest().log(DigestItem::as_changes_trie_root).cloned();
if let Some(changes_trie_root) = maybe_changes_trie_root {
if let Some(changes_trie_update) = operation.changes_trie_update {
self.changes_trie_storage.0.insert(
*header.number(),
changes_trie_root,
changes_trie_update
);
}
}
self.blockchain.insert(hash, header, justification, body, pending_block.state)?;
}
@@ -688,8 +666,8 @@ impl<Block: BlockT> backend::Backend<Block> for Backend<Block> where Block::Hash
None
}
fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage> {
Some(&self.changes_trie_storage)
fn changes_trie_storage(&self) -> Option<&dyn backend::PrunableStateChangesTrieStorage<Block>> {
None
}
fn offchain_storage(&self) -> Option<Self::OffchainStorage> {
@@ -737,66 +715,6 @@ impl<Block: BlockT> backend::RemoteBackend<Block> for Backend<Block> where Block
}
}
/// Prunable in-memory changes trie storage.
pub struct ChangesTrieStorage<Block: BlockT>(
InMemoryChangesTrieStorage<HasherFor<Block>, NumberFor<Block>>
);
impl<Block: BlockT> backend::PrunableStateChangesTrieStorage<Block> for ChangesTrieStorage<Block> {
fn oldest_changes_trie_block(
&self,
_config: &ChangesTrieConfiguration,
_best_finalized: NumberFor<Block>,
) -> NumberFor<Block> {
Zero::zero()
}
}
impl<Block: BlockT> sp_state_machine::ChangesTrieRootsStorage<HasherFor<Block>, NumberFor<Block>> for
ChangesTrieStorage<Block>
{
fn build_anchor(
&self,
_hash: Block::Hash,
) -> Result<sp_state_machine::ChangesTrieAnchorBlockId<Block::Hash, NumberFor<Block>>, String> {
Err("Dummy implementation".into())
}
fn root(
&self,
_anchor: &ChangesTrieAnchorBlockId<Block::Hash, NumberFor<Block>>,
_block: NumberFor<Block>,
) -> Result<Option<Block::Hash>, String> {
Err("Dummy implementation".into())
}
}
impl<Block: BlockT> sp_state_machine::ChangesTrieStorage<HasherFor<Block>, NumberFor<Block>> for
ChangesTrieStorage<Block>
{
fn as_roots_storage(&self)
-> &dyn sp_state_machine::ChangesTrieRootsStorage<HasherFor<Block>, NumberFor<Block>>
{
self
}
fn with_cached_changed_keys(
&self,
_root: &Block::Hash,
_functor: &mut dyn FnMut(&HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>),
) -> bool {
false
}
fn get(
&self,
key: &Block::Hash,
prefix: Prefix,
) -> Result<Option<sp_state_machine::DBValue>, String> {
self.0.get(key, prefix)
}
}
/// Check that genesis storage is valid.
pub fn check_genesis_storage(storage: &Storage) -> sp_blockchain::Result<()> {
if storage.top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) {
substrate/client/src/light/backend.rs
+19 -5
View File
@@ -21,19 +21,22 @@ use std::collections::HashMap;
use std::sync::Arc;
use parking_lot::RwLock;
use sp_core::storage::{ChildInfo, OwnedChildInfo};
use codec::{Decode, Encode};
use sp_core::ChangesTrieConfiguration;
use sp_core::storage::{well_known_keys, ChildInfo, OwnedChildInfo};
use sp_core::offchain::storage::InMemOffchainStorage;
use sp_state_machine::{
Backend as StateBackend, TrieBackend, InMemoryBackend, ChangesTrieTransaction
};
use sp_runtime::{generic::BlockId, Justification, Storage};
use sp_runtime::traits::{Block as BlockT, NumberFor, Zero, Header, HasherFor};
use crate::in_mem::{self, check_genesis_storage};
use crate::in_mem::check_genesis_storage;
use sp_blockchain::{Error as ClientError, Result as ClientResult};
use sc_client_api::{
backend::{
AuxStore, Backend as ClientBackend, BlockImportOperation, RemoteBackend, NewBlockState,
StorageCollection, ChildStorageCollection,
StorageCollection, ChildStorageCollection, PrunableStateChangesTrieStorage,
},
blockchain::{
HeaderBackend as BlockchainHeaderBackend, well_known_cache_keys,
@@ -62,6 +65,7 @@ pub struct ImportOperation<Block: BlockT, S> {
finalized_blocks: Vec<BlockId<Block>>,
set_head: Option<BlockId<Block>>,
storage_update: Option<InMemoryBackend<HasherFor<Block>>>,
changes_trie_config_update: Option<Option<ChangesTrieConfiguration>>,
_phantom: std::marker::PhantomData<S>,
}
@@ -115,7 +119,6 @@ impl<S, Block> ClientBackend<Block> for Backend<S, HasherFor<Block>>
type BlockImportOperation = ImportOperation<Block, S>;
type Blockchain = Blockchain<S>;
type State = GenesisOrUnavailableState<HasherFor<Block>>;
type ChangesTrieStorage = in_mem::ChangesTrieStorage<Block>;
type OffchainStorage = InMemOffchainStorage;
fn begin_operation(&self) -> ClientResult<Self::BlockImportOperation> {
@@ -127,6 +130,7 @@ impl<S, Block> ClientBackend<Block> for Backend<S, HasherFor<Block>>
finalized_blocks: Vec::new(),
set_head: None,
storage_update: None,
changes_trie_config_update: None,
_phantom: Default::default(),
})
}
@@ -148,6 +152,9 @@ impl<S, Block> ClientBackend<Block> for Backend<S, HasherFor<Block>>
if let Some(header) = operation.header {
let is_genesis_import = header.number().is_zero();
if let Some(new_config) = operation.changes_trie_config_update {
operation.cache.insert(well_known_cache_keys::CHANGES_TRIE_CONFIG, new_config.encode());
}
self.blockchain.storage().import_header(
header,
operation.cache,
@@ -194,7 +201,7 @@ impl<S, Block> ClientBackend<Block> for Backend<S, HasherFor<Block>>
self.blockchain.storage().usage_info()
}
fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage> {
fn changes_trie_storage(&self) -> Option<&dyn PrunableStateChangesTrieStorage<Block>> {
None
}
@@ -296,6 +303,13 @@ impl<S, Block> BlockImportOperation<Block> for ImportOperation<Block, S>
fn reset_storage(&mut self, input: Storage) -> ClientResult<Block::Hash> {
check_genesis_storage(&input)?;
// changes trie configuration
let changes_trie_config = input.top.iter()
.find(|(k, _)| &k[..] == well_known_keys::CHANGES_TRIE_CONFIG)
.map(|(_, v)| Decode::decode(&mut &v[..])
.expect("changes trie configuration is encoded properly at genesis"));
self.changes_trie_config_update = Some(changes_trie_config);
// this is only called when genesis block is imported => shouldn't be performance bottleneck
let mut storage: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, _> = HashMap::new();
storage.insert(None, input.top);
substrate/client/src/light/call_executor.rs
+2 -2
View File
@@ -259,7 +259,7 @@ fn check_execution_proof_with_make_header<Header, E, H, MakeNextHeader: Fn(&Head
let mut changes = OverlayedChanges::default();
let trie_backend = create_proof_check_backend(root, remote_proof)?;
let next_header = make_next_header(&request.header);
execution_proof_check_on_trie_backend::<H, _>(
execution_proof_check_on_trie_backend::<H, Header::Number, _>(
&trie_backend,
&mut changes,
executor,
@@ -268,7 +268,7 @@ fn check_execution_proof_with_make_header<Header, E, H, MakeNextHeader: Fn(&Head
)?;
// execute method
execution_proof_check_on_trie_backend::<H, _>(
execution_proof_check_on_trie_backend::<H, Header::Number, _>(
&trie_backend,
&mut changes,
executor,
substrate/client/src/light/fetcher.rs
+50 -31
View File
@@ -25,12 +25,12 @@ use codec::{Decode, Encode};
use sp_core::{convert_hash, traits::CodeExecutor};
use sp_runtime::traits::{
Block as BlockT, Header as HeaderT, Hash, HashFor, NumberFor,
SimpleArithmetic, CheckedConversion, Zero,
SimpleArithmetic, CheckedConversion,
};
use sp_state_machine::{
ChangesTrieRootsStorage, ChangesTrieAnchorBlockId, ChangesTrieConfigurationRange,
TrieBackend, read_proof_check, key_changes_proof_check, create_proof_check_backend_storage,
read_child_proof_check,
InMemoryChangesTrieStorage, TrieBackend, read_proof_check, key_changes_proof_check_with_db,
create_proof_check_backend_storage, read_child_proof_check,
};
pub use sp_state_machine::StorageProof;
use sp_blockchain::{Error as ClientError, Result as ClientResult};
@@ -113,30 +113,34 @@ impl<E, H, B: BlockT, S: BlockchainStorage<B>> LightDataChecker<E, H, B, S> {
)?;
}
// FIXME: remove this in https://github.com/paritytech/substrate/pull/3201
let changes_trie_config_range = ChangesTrieConfigurationRange {
config: &request.changes_trie_config,
zero: Zero::zero(),
end: None,
};
// and now check the key changes proof + get the changes
key_changes_proof_check::<H, _>(
changes_trie_config_range,
&RootsStorage {
roots: (request.tries_roots.0, &request.tries_roots.2),
prev_roots: remote_roots,
},
remote_proof,
request.first_block.0,
&ChangesTrieAnchorBlockId {
hash: convert_hash(&request.last_block.1),
number: request.last_block.0,
},
remote_max_block,
request.storage_key.as_ref().map(Vec::as_slice),
&request.key)
.map_err(|err| ClientError::ChangesTrieAccessFailed(err))
let mut result = Vec::new();
let proof_storage = InMemoryChangesTrieStorage::with_proof(remote_proof);
for config_range in &request.changes_trie_configs {
let result_range = key_changes_proof_check_with_db::<H, _>(
ChangesTrieConfigurationRange {
config: config_range.config.as_ref().ok_or(ClientError::ChangesTriesNotSupported)?,
zero: config_range.zero.0,
end: config_range.end.map(|(n, _)| n),
},
&RootsStorage {
roots: (request.tries_roots.0, &request.tries_roots.2),
prev_roots: &remote_roots,
},
&proof_storage,
request.first_block.0,
&ChangesTrieAnchorBlockId {
hash: convert_hash(&request.last_block.1),
number: request.last_block.0,
},
remote_max_block,
request.storage_key.as_ref().map(Vec::as_slice),
&request.key)
.map_err(|err| ClientError::ChangesTrieAccessFailed(err))?;
result.extend(result_range);
}
Ok(result)
}
/// Check CHT-based proof for changes tries roots.
@@ -284,7 +288,7 @@ impl<E, Block, H, S> FetchChecker<Block> for LightDataChecker<E, H, Block, S>
/// A view of BTreeMap<Number, Hash> as a changes trie roots storage.
struct RootsStorage<'a, Number: SimpleArithmetic, Hash: 'a> {
roots: (Number, &'a [Hash]),
prev_roots: BTreeMap<Number, Hash>,
prev_roots: &'a BTreeMap<Number, Hash>,
}
impl<'a, H, Number, Hash> ChangesTrieRootsStorage<H, Number> for RootsStorage<'a, Number, Hash>
@@ -340,7 +344,7 @@ pub mod tests {
use crate::in_mem::{Blockchain as InMemoryBlockchain};
use crate::light::fetcher::{FetchChecker, LightDataChecker, RemoteHeaderRequest};
use crate::light::blockchain::tests::{DummyStorage, DummyBlockchain};
use sp_core::{blake2_256, Blake2Hasher, H256};
use sp_core::{blake2_256, Blake2Hasher, ChangesTrieConfiguration, H256};
use sp_core::storage::{well_known_keys, StorageKey, ChildInfo};
use sp_runtime::generic::BlockId;
use sp_state_machine::Backend;
@@ -569,8 +573,13 @@ pub mod tests {
// check proof on local client
let local_roots_range = local_roots.clone()[(begin - 1) as usize..].to_vec();
let config = ChangesTrieConfiguration::new(4, 2);
let request = RemoteChangesRequest::<Header> {
changes_trie_config: runtime::changes_trie_config(),
changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange {
zero: (0, Default::default()),
end: None,
config: Some(config),
}],
first_block: (begin, begin_hash),
last_block: (end, end_hash),
max_block: (max, max_hash),
@@ -624,8 +633,13 @@ pub mod tests {
);
// check proof on local client
let config = ChangesTrieConfiguration::new(4, 2);
let request = RemoteChangesRequest::<Header> {
changes_trie_config: runtime::changes_trie_config(),
changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange {
zero: (0, Default::default()),
end: None,
config: Some(config),
}],
first_block: (1, b1),
last_block: (4, b4),
max_block: (4, b4),
@@ -665,8 +679,13 @@ pub mod tests {
begin_hash, end_hash, begin_hash, max_hash, None, &key).unwrap();
let local_roots_range = local_roots.clone()[(begin - 1) as usize..].to_vec();
let config = ChangesTrieConfiguration::new(4, 2);
let request = RemoteChangesRequest::<Header> {
changes_trie_config: runtime::changes_trie_config(),
changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange {
zero: (0, Default::default()),
end: None,
config: Some(config),
}],
first_block: (begin, begin_hash),
last_block: (end, end_hash),
max_block: (max, max_hash),