pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-11 09:31:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow updating configuration of changes tries (#3201)

Browse Source 
* DigestItem::ChangesTrieSignal

* introduce changes_trie::State

* introduce config activation block

* ChangesTrieSignal::as_new_configuration

* moved well_known_cache_keys to client

* extracted DbChangesTrieStorage to separate file

* change meaning of none in blockchain cache

* changes trie config (FULL) cache draft

* eliminating const ChangesTrieConfiguration

* delay pruning

* continue elimination

* do not prune CT config from cache

* removed redundant code

* fix some TODOs

* introduce ConfigurationRange

* use Configuration range in build

* build skewed digest

* remove debug print

* extracted surface iterator

* key_changes works with skewed digests

* fix client build

* add test for NeverPrune

* fix TODO

* fixed some TODOs

* more tests

* fixing TODOs

* fixed compilation

* update runtime version

* git rid of large tuple

* too long lines

* config_activation_block -> zero

* obsolete TODO

* removed unjustified expect

* update TODOs with issue number

* new CT pruning algorithm

fixed cache + multiple blocks finalization

track CT configuraiton on light clients

support CT configuration change revert

revert CT config test

new CT pruning algorithm

fixed cache + multiple blocks finalization

track CT configuraiton on light clients

support CT configuration change revert

revert CT config test

* BlockIdOrHeader isn't really required

* removed debug leftovers + some docs

* more docs

* more post-merge fixes

* more post-merge fixes

* revertes some unnecessary changes

* reverted unnecessary changes

* fix compilation + unnecessary changes

* (restart CI)

* fix cache update when finalizing multiple blocks

* fixed tests

* collect_extrinsics -> set_collect_extrinsics

* restore lost test

* do not calculate block number twice

* Update primitives/blockchain/src/error.rs

Co-Authored-By: cheme <emericchevalier.pro@gmail.com>

* map_err -> unwrap_or

* document get_at Result

* delete abandoned file

* added weight for set_changes_trie_config

* prefer_configs -> fail_if_disabled

* Update client/api/src/backend.rs

Co-Authored-By: cheme <emericchevalier.pro@gmail.com>

* Update client/db/src/changes_tries_storage.rs

Co-Authored-By: cheme <emericchevalier.pro@gmail.com>

* CommitOperation+merge -> CommitOperations

* fixed test compilation

* merged two different CTRange structs

* lost file

* uggrade db from v0 to v1 (init CT cache + add column)

* fix after merge

Co-authored-by: cheme <emericchevalier.pro@gmail.com>
Co-authored-by: Gavin Wood <github@gavwood.com>
This commit is contained in:
Svyatoslav Nikolsky
2020-01-16 19:38:24 +03:00
committed by Gavin Wood
parent 45fbf09dac
commit febf29390a
48 changed files with 2743 additions and 1548 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/Cargo.lock
Generated
+11
View File
@@ -5231,6 +5231,7 @@ dependencies = [
"sp-state-machine 0.8.0",
"sp-trie 2.0.0",
"substrate-test-runtime-client 2.0.0",
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -7075,6 +7076,15 @@ name = "target_info"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "tempdir"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rand 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
"remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tempfile"
version = "3.1.0"
@@ -8698,6 +8708,7 @@ dependencies = [
"checksum take_mut 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60"
"checksum target-lexicon 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6f4c118a7a38378f305a9e111fcb2f7f838c0be324bfb31a77ea04f7f6e684b4"
"checksum target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c63f48baada5c52e65a29eef93ab4f8982681b67f9e8d29c7b05abcfec2b9ffe"
"checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
"checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
"checksum termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "96d6098003bde162e4277c70665bd87c326f5a0c3f3fbfb285787fa482d54e6e"
"checksum test-case 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a605baa797821796a751f4a959e1206079b24a4b7e1ed302b7d785d81a9276c9"
substrate/client/api/src/backend.rs
+30 -11
View File
@@ -18,11 +18,11 @@
use std::sync::Arc;
use std::collections::HashMap;
use sp_core::ChangesTrieConfiguration;
use sp_core::ChangesTrieConfigurationRange;
use sp_core::offchain::OffchainStorage;
use sp_runtime::{generic::BlockId, Justification, Storage};
use sp_runtime::traits::{Block as BlockT, NumberFor, HasherFor};
use sp_state_machine::{ChangesTrieStorage as StateChangesTrieStorage, ChangesTrieTransaction};
use sp_state_machine::{ChangesTrieState, ChangesTrieStorage as StateChangesTrieStorage, ChangesTrieTransaction};
use crate::{
blockchain::{
Backend as BlockchainBackend, well_known_cache_keys
@@ -248,8 +248,6 @@ pub trait Backend<Block: BlockT>: AuxStore + Send + Sync {
type Blockchain: BlockchainBackend<Block>;
/// Associated state backend type.
type State: StateBackend<HasherFor<Block>> + Send;
/// Changes trie storage.
type ChangesTrieStorage: PrunableStateChangesTrieStorage<Block>;
/// Offchain workers local storage.
type OffchainStorage: OffchainStorage;
@@ -284,7 +282,7 @@ pub trait Backend<Block: BlockT>: AuxStore + Send + Sync {
fn usage_info(&self) -> Option<UsageInfo>;
/// Returns reference to changes trie storage.
fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage>;
fn changes_trie_storage(&self) -> Option<&dyn PrunableStateChangesTrieStorage<Block>>;
/// Returns a handle to offchain storage.
fn offchain_storage(&self) -> Option<Self::OffchainStorage>;
@@ -342,12 +340,16 @@ pub trait Backend<Block: BlockT>: AuxStore + Send + Sync {
pub trait PrunableStateChangesTrieStorage<Block: BlockT>:
StateChangesTrieStorage<HasherFor<Block>, NumberFor<Block>>
{
/// Get number block of oldest, non-pruned changes trie.
fn oldest_changes_trie_block(
&self,
config: &ChangesTrieConfiguration,
best_finalized: NumberFor<Block>,
) -> NumberFor<Block>;
/// Get reference to StateChangesTrieStorage.
fn storage(&self) -> &dyn StateChangesTrieStorage<HasherFor<Block>, NumberFor<Block>>;
/// Get configuration at given block.
fn configuration_at(&self, at: &BlockId<Block>) -> sp_blockchain::Result<
ChangesTrieConfigurationRange<NumberFor<Block>, Block::Hash>
>;
/// Get end block (inclusive) of oldest pruned max-level (or skewed) digest trie blocks range.
/// It is guaranteed that we have no any changes tries before (and including) this block.
/// It is guaranteed that all existing changes tries after this block are not yet pruned (if created).
fn oldest_pruned_digest_range_end(&self) -> NumberFor<Block>;
}
/// Mark for all Backend implementations, that are making use of state data, stored locally.
@@ -364,3 +366,20 @@ pub trait RemoteBackend<Block: BlockT>: Backend<Block> {
/// locally, or prepares request to fetch that data from remote node.
fn remote_blockchain(&self) -> Arc<dyn RemoteBlockchain<Block>>;
}
/// Return changes tries state at given block.
pub fn changes_tries_state_at_block<'a, Block: BlockT>(
block: &BlockId<Block>,
maybe_storage: Option<&'a dyn PrunableStateChangesTrieStorage<Block>>,
) -> sp_blockchain::Result<Option<ChangesTrieState<'a, HasherFor<Block>, NumberFor<Block>>>> {
let storage = match maybe_storage {
Some(storage) => storage,
None => return Ok(None),
};
let config_range = storage.configuration_at(block)?;
match config_range.config {
Some(config) => Ok(Some(ChangesTrieState::new(config, config_range.zero.0, storage.storage()))),
None => Ok(None),
}
}
substrate/client/api/src/light.rs
+3 -3
View File
@@ -26,7 +26,7 @@ use sp_runtime::{
},
generic::BlockId
};
use sp_core::ChangesTrieConfiguration;
use sp_core::ChangesTrieConfigurationRange;
use sp_state_machine::StorageProof;
use sp_blockchain::{
HeaderMetadata, well_known_cache_keys, HeaderBackend, Cache as BlockchainCache,
@@ -96,8 +96,8 @@ pub struct RemoteReadChildRequest<Header: HeaderT> {
/// Remote key changes read request.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RemoteChangesRequest<Header: HeaderT> {
/// Changes trie configuration.
pub changes_trie_config: ChangesTrieConfiguration,
/// All changes trie configurations that are valid within [first_block; last_block].
pub changes_trie_configs: Vec<ChangesTrieConfigurationRange<Header::Number, Header::Hash>>,
/// Query changes from range of blocks, starting (and including) with this hash...
pub first_block: (Header::Number, Header::Hash),
/// ...ending (and including) with this hash. Should come after first_block and
substrate/client/authority-discovery/src/tests.rs
+2 -4
View File
@@ -212,12 +212,10 @@ impl ApiExt<Block> for RuntimeApi {
unimplemented!("Not required for testing!")
}
fn into_storage_changes<
T: sp_api::ChangesTrieStorage<sp_api::HasherFor<Block>, sp_api::NumberFor<Block>>
>(
fn into_storage_changes(
&self,
_: &Self::StateBackend,
_: Option<&T>,
_: Option<&sp_api::ChangesTrieState<sp_api::HasherFor<Block>, sp_api::NumberFor<Block>>>,
_: <Block as sp_api::BlockT>::Hash,
) -> std::result::Result<sp_api::StorageChanges<Self::StateBackend, Block>, String>
where Self: Sized
substrate/client/basic-authorship/src/basic_authorship.rs
+5 -2
View File
@@ -374,9 +374,12 @@ mod tests {
api.execute_block(&block_id, proposal.block).unwrap();
let state = backend.state_at(block_id).unwrap();
let changes_trie_storage = backend.changes_trie_storage();
let changes_trie_state = backend::changes_tries_state_at_block(
&block_id,
backend.changes_trie_storage(),
).unwrap();
let storage_changes = api.into_storage_changes(&state, changes_trie_storage, genesis_hash)
let storage_changes = api.into_storage_changes(&state, changes_trie_state.as_ref(), genesis_hash)
.unwrap();
assert_eq!(
substrate/client/block-builder/src/lib.rs
+5 -2
View File
@@ -191,14 +191,17 @@ where
let proof = self.api.extract_proof();
let state = self.backend.state_at(self.block_id)?;
let changes_trie_storage = self.backend.changes_trie_storage();
let changes_trie_state = backend::changes_tries_state_at_block(
&self.block_id,
self.backend.changes_trie_storage(),
)?;
let parent_hash = self.parent_hash;
// The unsafe is required because the consume requires that we drop/consume the inner api
// (what we do here).
let storage_changes = self.api.into_storage_changes(
&state,
changes_trie_storage,
changes_trie_state.as_ref(),
parent_hash,
);
substrate/client/consensus/aura/src/lib.rs
+11 -8
View File
@@ -675,19 +675,21 @@ fn initialize_authorities_cache<A, B, C>(client: &C) -> Result<(), ConsensusErro
};
// check if we already have initialized the cache
let genesis_id = BlockId::Number(Zero::zero());
let genesis_authorities: Option<Vec<A>> = cache
.get_at(&well_known_cache_keys::AUTHORITIES, &genesis_id)
.and_then(|(_, _, v)| Decode::decode(&mut &v[..]).ok());
if genesis_authorities.is_some() {
return Ok(());
}
let map_err = |error| sp_consensus::Error::from(sp_consensus::Error::ClientImport(
format!(
"Error initializing authorities cache: {}",
error,
)));
let genesis_id = BlockId::Number(Zero::zero());
let genesis_authorities: Option<Vec<A>> = cache
.get_at(&well_known_cache_keys::AUTHORITIES, &genesis_id)
.unwrap_or(None)
.and_then(|(_, _, v)| Decode::decode(&mut &v[..]).ok());
if genesis_authorities.is_some() {
return Ok(());
}
let genesis_authorities = authorities(client, &genesis_id)?;
cache.initialize(&well_known_cache_keys::AUTHORITIES, genesis_authorities.encode())
.map_err(map_err)?;
@@ -706,6 +708,7 @@ fn authorities<A, B, C>(client: &C, at: &BlockId<B>) -> Result<Vec<A>, Consensus
.cache()
.and_then(|cache| cache
.get_at(&well_known_cache_keys::AUTHORITIES, at)
.unwrap_or(None)
.and_then(|(_, _, v)| Decode::decode(&mut &v[..]).ok())
)
.or_else(|| AuraApi::authorities(&*client.runtime_api(), at).ok())
substrate/client/db/Cargo.toml
+2
View File
@@ -31,6 +31,8 @@ sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" }
substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" }
env_logger = "0.7.0"
quickcheck = "0.9"
kvdb-rocksdb = "0.4"
tempdir = "0.3"
[features]
default = []
substrate/client/db/src/cache/list_cache.rs
Vendored
+392 -157
View File
@@ -94,6 +94,12 @@ pub enum CommitOperation<Block: BlockT, T: CacheItemT> {
BlockReverted(BTreeMap<usize, Option<Fork<Block, T>>>),
}
/// A set of commit operations.
#[derive(Debug)]
pub struct CommitOperations<Block: BlockT, T: CacheItemT> {
operations: Vec<CommitOperation<Block, T>>,
}
/// Single fork of list-based cache.
#[derive(Debug)]
#[cfg_attr(test, derive(PartialEq))]
@@ -123,21 +129,17 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
storage: S,
pruning_strategy: PruningStrategy<NumberFor<Block>>,
best_finalized_block: ComplexBlockId<Block>,
) -> Self {
) -> ClientResult<Self> {
let (best_finalized_entry, unfinalized) = storage.read_meta()
.and_then(|meta| read_forks(&storage, meta))
.unwrap_or_else(|error| {
warn!(target: "db", "Unable to initialize list cache: {}. Restarting", error);
(None, Vec::new())
});
.and_then(|meta| read_forks(&storage, meta))?;
ListCache {
Ok(ListCache {
storage,
pruning_strategy,
best_finalized_block,
best_finalized_entry,
unfinalized,
}
})
}
/// Get reference to the storage.
@@ -145,6 +147,12 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
&self.storage
}
/// Get unfinalized forks reference.
#[cfg(test)]
pub fn unfinalized(&self) -> &[Fork<Block, T>] {
&self.unfinalized
}
/// Get value valid at block.
pub fn value_at_block(
&self,
@@ -156,8 +164,8 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
// BUT since we're not guaranteeing to provide correct values for forks
// behind the finalized block, check if the block is finalized first
if !chain::is_finalized_block(&self.storage, at, Bounded::max_value())? {
return Ok(None);
if !chain::is_finalized_block(&self.storage, &at, Bounded::max_value())? {
return Err(ClientError::NotInFinalizedChain);
}
self.best_finalized_entry.as_ref()
@@ -171,11 +179,14 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
// IF there's no matching fork, ensure that this isn't a block from a fork that has forked
// behind the best finalized block and search at finalized fork
match self.find_unfinalized_fork(at)? {
match self.find_unfinalized_fork(&at)? {
Some(fork) => Some(&fork.head),
None => match self.best_finalized_entry.as_ref() {
Some(best_finalized_entry) if chain::is_connected_to_block(&self.storage, &best_finalized_entry.valid_from, at)? =>
Some(best_finalized_entry),
Some(best_finalized_entry) if chain::is_connected_to_block(
&self.storage,
&at,
&best_finalized_entry.valid_from,
)? => Some(best_finalized_entry),
_ => None,
},
}
@@ -198,9 +209,98 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
block: ComplexBlockId<Block>,
value: Option<T>,
entry_type: EntryType,
operations: &mut CommitOperations<Block, T>,
) -> ClientResult<()> {
Ok(operations.append(self.do_on_block_insert(tx, parent, block, value, entry_type, operations)?))
}
/// When previously inserted block is finalized.
pub fn on_block_finalize<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>,
operations: &mut CommitOperations<Block, T>,
) -> ClientResult<()> {
Ok(operations.append(self.do_on_block_finalize(tx, parent, block, operations)?))
}
/// When block is reverted.
pub fn on_block_revert<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
reverted_block: &ComplexBlockId<Block>,
operations: &mut CommitOperations<Block, T>,
) -> ClientResult<()> {
Ok(operations.append(Some(self.do_on_block_revert(tx, reverted_block)?)))
}
/// When transaction is committed.
pub fn on_transaction_commit(&mut self, ops: CommitOperations<Block, T>) {
for op in ops.operations {
match op {
CommitOperation::AppendNewBlock(index, best_block) => {
let mut fork = self.unfinalized.get_mut(index)
.expect("ListCache is a crate-private type;
internal clients of ListCache are committing transaction while cache is locked;
CommitOperation holds valid references while cache is locked; qed");
fork.best_block = Some(best_block);
},
CommitOperation::AppendNewEntry(index, entry) => {
let mut fork = self.unfinalized.get_mut(index)
.expect("ListCache is a crate-private type;
internal clients of ListCache are committing transaction while cache is locked;
CommitOperation holds valid references while cache is locked; qed");
fork.best_block = Some(entry.valid_from.clone());
fork.head = entry;
},
CommitOperation::AddNewFork(entry) => {
self.unfinalized.push(Fork {
best_block: Some(entry.valid_from.clone()),
head: entry,
});
},
CommitOperation::BlockFinalized(block, finalizing_entry, forks) => {
self.best_finalized_block = block;
if let Some(finalizing_entry) = finalizing_entry {
self.best_finalized_entry = Some(finalizing_entry);
}
for fork_index in forks.iter().rev() {
self.unfinalized.remove(*fork_index);
}
},
CommitOperation::BlockReverted(forks) => {
for (fork_index, updated_fork) in forks.into_iter().rev() {
match updated_fork {
Some(updated_fork) => self.unfinalized[fork_index] = updated_fork,
None => { self.unfinalized.remove(fork_index); },
}
}
},
}
}
}
fn do_on_block_insert<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>,
value: Option<T>,
entry_type: EntryType,
operations: &CommitOperations<Block, T>,
) -> ClientResult<Option<CommitOperation<Block, T>>> {
// this guarantee is currently provided by LightStorage && we're relying on it here
debug_assert!(entry_type != EntryType::Final || self.best_finalized_block.hash == parent.hash);
let prev_operation = operations.operations.last();
debug_assert!(
entry_type != EntryType::Final ||
self.best_finalized_block.hash == parent.hash ||
match prev_operation {
Some(&CommitOperation::BlockFinalized(ref best_finalized_block, _, _))
=> best_finalized_block.hash == parent.hash,
_ => false,
}
);
// we do not store any values behind finalized
if block.number != Zero::zero() && self.best_finalized_block.number >= block.number {
@@ -296,7 +396,7 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
}
// cleanup database from abandoned unfinalized forks and obsolete finalized entries
let abandoned_forks = self.destroy_abandoned_forks(tx, &block);
let abandoned_forks = self.destroy_abandoned_forks(tx, &block, prev_operation);
self.prune_finalized_entries(tx, &block);
match new_storage_entry {
@@ -310,34 +410,39 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
}
}
/// When previously inserted block is finalized.
pub fn on_block_finalize<Tx: StorageTransaction<Block, T>>(
fn do_on_block_finalize<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>,
operations: &CommitOperations<Block, T>,
) -> ClientResult<Option<CommitOperation<Block, T>>> {
// this guarantee is currently provided by LightStorage && we're relying on it here
debug_assert_eq!(self.best_finalized_block.hash, parent.hash);
// this guarantee is currently provided by db backend && we're relying on it here
let prev_operation = operations.operations.last();
debug_assert!(
self.best_finalized_block.hash == parent.hash ||
match prev_operation {
Some(&CommitOperation::BlockFinalized(ref best_finalized_block, _, _))
=> best_finalized_block.hash == parent.hash,
_ => false,
}
);
// there could be at most one entry that is finalizing
let finalizing_entry = self.storage.read_entry(&block)?
.map(|entry| entry.into_entry(block.clone()));
// cleanup database from abandoned unfinalized forks and obsolete finalized entries
let abandoned_forks = self.destroy_abandoned_forks(tx, &block);
let abandoned_forks = self.destroy_abandoned_forks(tx, &block, prev_operation);
self.prune_finalized_entries(tx, &block);
let update_meta = finalizing_entry.is_some();
let operation = CommitOperation::BlockFinalized(block, finalizing_entry, abandoned_forks);
if update_meta {
tx.update_meta(self.best_finalized_entry.as_ref(), &self.unfinalized, &operation);
}
tx.update_meta(self.best_finalized_entry.as_ref(), &self.unfinalized, &operation);
Ok(Some(operation))
}
/// When block is reverted.
pub fn on_block_revert<Tx: StorageTransaction<Block, T>>(
fn do_on_block_revert<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
reverted_block: &ComplexBlockId<Block>,
@@ -374,50 +479,6 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
Ok(operation)
}
/// When transaction is committed.
pub fn on_transaction_commit(&mut self, op: CommitOperation<Block, T>) {
match op {
CommitOperation::AppendNewBlock(index, best_block) => {
let mut fork = self.unfinalized.get_mut(index)
.expect("ListCache is a crate-private type;
internal clients of ListCache are committing transaction while cache is locked;
CommitOperation holds valid references while cache is locked; qed");
fork.best_block = Some(best_block);
},
CommitOperation::AppendNewEntry(index, entry) => {
let mut fork = self.unfinalized.get_mut(index)
.expect("ListCache is a crate-private type;
internal clients of ListCache are committing transaction while cache is locked;
CommitOperation holds valid references while cache is locked; qed");
fork.best_block = Some(entry.valid_from.clone());
fork.head = entry;
},
CommitOperation::AddNewFork(entry) => {
self.unfinalized.push(Fork {
best_block: Some(entry.valid_from.clone()),
head: entry,
});
},
CommitOperation::BlockFinalized(block, finalizing_entry, forks) => {
self.best_finalized_block = block;
if let Some(finalizing_entry) = finalizing_entry {
self.best_finalized_entry = Some(finalizing_entry);
}
for fork_index in forks.iter().rev() {
self.unfinalized.remove(*fork_index);
}
},
CommitOperation::BlockReverted(forks) => {
for (fork_index, updated_fork) in forks.into_iter().rev() {
match updated_fork {
Some(updated_fork) => self.unfinalized[fork_index] = updated_fork,
None => { self.unfinalized.remove(fork_index); },
}
}
},
}
}
/// Prune old finalized entries.
fn prune_finalized_entries<Tx: StorageTransaction<Block, T>>(
&self,
@@ -475,10 +536,22 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
fn destroy_abandoned_forks<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
block: &ComplexBlockId<Block>
block: &ComplexBlockId<Block>,
prev_operation: Option<&CommitOperation<Block, T>>,
) -> BTreeSet<usize> {
let mut destroyed = BTreeSet::new();
for (index, fork) in self.unfinalized.iter().enumerate() {
// if some block has been finalized already => take it into account
let prev_abandoned_forks = match prev_operation {
Some(&CommitOperation::BlockFinalized(_, _, ref abandoned_forks)) => Some(abandoned_forks),
_ => None,
};
let mut destroyed = prev_abandoned_forks.cloned().unwrap_or_else(|| BTreeSet::new());
let live_unfinalized = self.unfinalized.iter()
.enumerate()
.filter(|(idx, _)| prev_abandoned_forks
.map(|prev_abandoned_forks| !prev_abandoned_forks.contains(idx))
.unwrap_or(true));
for (index, fork) in live_unfinalized {
if fork.head.valid_from.number == block.number {
destroyed.insert(index);
if fork.head.valid_from.hash != block.hash {
@@ -493,7 +566,10 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
}
/// Search unfinalized fork where given block belongs.
fn find_unfinalized_fork(&self, block: &ComplexBlockId<Block>) -> ClientResult<Option<&Fork<Block, T>>> {
fn find_unfinalized_fork(
&self,
block: &ComplexBlockId<Block>,
) -> ClientResult<Option<&Fork<Block, T>>> {
for unfinalized in &self.unfinalized {
if unfinalized.matches(&self.storage, block)? {
return Ok(Some(&unfinalized));
@@ -549,7 +625,7 @@ impl<Block: BlockT, T: CacheItemT> Fork<Block, T> {
};
// check if the parent is connected to the beginning of the range
if !chain::is_connected_to_block(storage, &parent, &begin)? {
if !chain::is_connected_to_block(storage, parent, &begin)? {
return Ok(None);
}
@@ -621,6 +697,65 @@ impl<Block: BlockT, T: CacheItemT> Fork<Block, T> {
}
}
impl<Block: BlockT, T: CacheItemT> Default for CommitOperations<Block, T> {
fn default() -> Self {
CommitOperations { operations: Vec::new() }
}
}
// This should never be allowed for non-test code to avoid revealing its internals.
#[cfg(test)]
impl<Block: BlockT, T: CacheItemT> From<Vec<CommitOperation<Block, T>>> for CommitOperations<Block, T> {
fn from(operations: Vec<CommitOperation<Block, T>>) -> Self {
CommitOperations { operations }
}
}
impl<Block: BlockT, T: CacheItemT> CommitOperations<Block, T> {
/// Append operation to the set.
fn append(&mut self, new_operation: Option<CommitOperation<Block, T>>) {
let new_operation = match new_operation {
Some(new_operation) => new_operation,
None => return,
};
let last_operation = match self.operations.pop() {
Some(last_operation) => last_operation,
None => {
self.operations.push(new_operation);
return;
},
};
// we are able (and obliged to) to merge two consequent block finalization operations
match last_operation {
CommitOperation::BlockFinalized(old_finalized_block, old_finalized_entry, old_abandoned_forks) => {
match new_operation {
CommitOperation::BlockFinalized(new_finalized_block, new_finalized_entry, new_abandoned_forks) => {
self.operations.push(CommitOperation::BlockFinalized(
new_finalized_block,
new_finalized_entry,
new_abandoned_forks,
));
},
_ => {
self.operations.push(CommitOperation::BlockFinalized(
old_finalized_block,
old_finalized_entry,
old_abandoned_forks,
));
self.operations.push(new_operation);
},
}
},
_ => {
self.operations.push(last_operation);
self.operations.push(new_operation);
},
}
}
}
/// Destroy fork by deleting all unfinalized entries.
pub fn destroy_fork<Block: BlockT, T: CacheItemT, S: Storage<Block, T>, Tx: StorageTransaction<Block, T>>(
head_valid_from: ComplexBlockId<Block>,
@@ -674,7 +809,7 @@ mod chain {
block1: &ComplexBlockId<Block>,
block2: &ComplexBlockId<Block>,
) -> ClientResult<bool> {
let (begin, end) = if block1 > block2 { (block2, block1) } else { (block1, block2) };
let (begin, end) = if *block1 > *block2 { (block2, block1) } else { (block1, block2) };
let mut current = storage.read_header(&end.hash)?
.ok_or_else(|| ClientError::UnknownBlock(format!("{}", end.hash)))?;
while *current.number() > begin.number {
@@ -773,8 +908,8 @@ pub mod tests {
// when block is earlier than best finalized block AND it is not finalized
// --- 50 ---
// ----------> [100]
assert_eq!(ListCache::<_, u64, _>::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
.value_at_block(&test_id(50)).unwrap(), None);
assert!(ListCache::<_, u64, _>::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
.unwrap().value_at_block(&test_id(50)).is_err());
// when block is earlier than best finalized block AND it is finalized AND value is some
// [30] ---- 50 ---> [100]
assert_eq!(ListCache::new(
@@ -784,7 +919,7 @@ pub mod tests {
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 })
.with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }),
PruningStrategy::ByDepth(1024), test_id(100)
).value_at_block(&test_id(50)).unwrap(), Some((test_id(30), Some(test_id(100)), 30)));
).unwrap().value_at_block(&test_id(50)).unwrap(), Some((test_id(30), Some(test_id(100)), 30)));
// when block is the best finalized block AND value is some
// ---> [100]
assert_eq!(ListCache::new(
@@ -794,18 +929,18 @@ pub mod tests {
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 })
.with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }),
PruningStrategy::ByDepth(1024), test_id(100)
).value_at_block(&test_id(100)).unwrap(), Some((test_id(100), None, 100)));
).unwrap().value_at_block(&test_id(100)).unwrap(), Some((test_id(100), None, 100)));
// when block is parallel to the best finalized block
// ---- 100
// ---> [100]
assert_eq!(ListCache::new(
assert!(ListCache::new(
DummyStorage::new()
.with_meta(Some(test_id(100)), Vec::new())
.with_id(50, H256::from_low_u64_be(50))
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 })
.with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }),
PruningStrategy::ByDepth(1024), test_id(100)
).value_at_block(&ComplexBlockId::new(H256::from_low_u64_be(2), 100)).unwrap(), None);
).unwrap().value_at_block(&ComplexBlockId::new(H256::from_low_u64_be(2), 100)).is_err());
// when block is later than last finalized block AND there are no forks AND finalized value is Some
// ---> [100] --- 200
@@ -815,7 +950,7 @@ pub mod tests {
.with_id(50, H256::from_low_u64_be(50))
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 }),
PruningStrategy::ByDepth(1024), test_id(100)
).value_at_block(&test_id(200)).unwrap(), Some((test_id(100), None, 100)));
).unwrap().value_at_block(&test_id(200)).unwrap(), Some((test_id(100), None, 100)));
// when block is later than last finalized block AND there are no matching forks
// AND block is connected to finalized block AND finalized value is Some
@@ -831,7 +966,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(fork_header(0, 2, 3)),
PruningStrategy::ByDepth(1024), test_id(2)
).value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
).unwrap().value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
// when block is later than last finalized block AND there are no matching forks
// AND block is not connected to finalized block
// --- 2 --- 3
@@ -848,7 +983,7 @@ pub mod tests {
.with_header(fork_header(0, 1, 3))
.with_header(fork_header(0, 1, 2)),
PruningStrategy::ByDepth(1024), test_id(2)
).value_at_block(&fork_id(0, 1, 3)).unwrap(), None);
).unwrap().value_at_block(&fork_id(0, 1, 3)).unwrap(), None);
// when block is later than last finalized block AND it appends to unfinalized fork from the end
// AND unfinalized value is Some
@@ -861,7 +996,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(test_header(5)),
PruningStrategy::ByDepth(1024), test_id(2)
).value_at_block(&correct_id(5)).unwrap(), Some((correct_id(4), None, 4)));
).unwrap().value_at_block(&correct_id(5)).unwrap(), Some((correct_id(4), None, 4)));
// when block is later than last finalized block AND it does not fits unfinalized fork
// AND it is connected to the finalized block AND finalized value is Some
// ---> [2] ----------> [4]
@@ -876,7 +1011,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(fork_header(0, 2, 3)),
PruningStrategy::ByDepth(1024), test_id(2)
).value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
).unwrap().value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
}
#[test]
@@ -885,22 +1020,25 @@ pub mod tests {
let fin = EntryType::Final;
// when trying to insert block < finalized number
assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
.on_block_insert(
let mut ops = Default::default();
assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100)).unwrap()
.do_on_block_insert(
&mut DummyTransaction::new(),
test_id(49),
test_id(50),
Some(50),
nfin,
&mut ops,
).unwrap().is_none());
// when trying to insert block @ finalized number
assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
.on_block_insert(
assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100)).unwrap()
.do_on_block_insert(
&mut DummyTransaction::new(),
test_id(99),
test_id(100),
Some(100),
nfin,
&Default::default(),
).unwrap().is_none());
// when trying to insert non-final block AND it appends to the best block of unfinalized fork
@@ -910,19 +1048,23 @@ pub mod tests {
.with_meta(None, vec![test_id(4)])
.with_entry(test_id(4), StorageEntry { prev_valid_from: None, value: 4 }),
PruningStrategy::ByDepth(1024), test_id(2)
);
).unwrap();
cache.unfinalized[0].best_block = Some(test_id(4));
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, test_id(4), test_id(5), Some(4), nfin).unwrap(),
Some(CommitOperation::AppendNewBlock(0, test_id(5))));
assert_eq!(
cache.do_on_block_insert(&mut tx, test_id(4), test_id(5), Some(4), nfin, &Default::default()).unwrap(),
Some(CommitOperation::AppendNewBlock(0, test_id(5))),
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
// when trying to insert non-final block AND it appends to the best block of unfinalized fork
// AND new value is the same as in the fork' best block
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, test_id(4), test_id(5), Some(5), nfin).unwrap(),
Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: test_id(5), value: 5 })));
assert_eq!(
cache.do_on_block_insert(&mut tx, test_id(4), test_id(5), Some(5), nfin, &Default::default()).unwrap(),
Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: test_id(5), value: 5 })),
);
assert_eq!(*tx.inserted_entries(), vec![test_id(5).hash].into_iter().collect());
assert!(tx.removed_entries().is_empty());
assert_eq!(*tx.updated_meta(), Some(Metadata { finalized: None, unfinalized: vec![test_id(5)] }));
@@ -935,18 +1077,36 @@ pub mod tests {
.with_entry(correct_id(4), StorageEntry { prev_valid_from: None, value: 4 })
.with_header(test_header(4)),
PruningStrategy::ByDepth(1024), test_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(4), correct_id(5), Some(4), nfin).unwrap(),
Some(CommitOperation::AppendNewBlock(0, correct_id(5))));
assert_eq!(
cache.do_on_block_insert(
&mut tx,
correct_id(4),
correct_id(5),
Some(4),
nfin,
&Default::default(),
).unwrap(),
Some(CommitOperation::AppendNewBlock(0, correct_id(5))),
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
// when trying to insert non-final block AND it is the first block that appends to the best block of unfinalized fork
// AND new value is the same as in the fork' best block
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(4), correct_id(5), Some(5), nfin).unwrap(),
Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(5), value: 5 })));
assert_eq!(
cache.do_on_block_insert(
&mut tx,
correct_id(4),
correct_id(5),
Some(5),
nfin,
&Default::default(),
).unwrap(),
Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(5), value: 5 })),
);
assert_eq!(*tx.inserted_entries(), vec![correct_id(5).hash].into_iter().collect());
assert!(tx.removed_entries().is_empty());
assert_eq!(*tx.updated_meta(), Some(Metadata { finalized: None, unfinalized: vec![correct_id(5)] }));
@@ -961,10 +1121,13 @@ pub mod tests {
.with_header(test_header(3))
.with_header(test_header(4)),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(3), fork_id(0, 3, 4), Some(14), nfin).unwrap(),
Some(CommitOperation::AddNewFork(Entry { valid_from: fork_id(0, 3, 4), value: 14 })));
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(3), fork_id(0, 3, 4), Some(14), nfin, &Default::default())
.unwrap(),
Some(CommitOperation::AddNewFork(Entry { valid_from: fork_id(0, 3, 4), value: 14 })),
);
assert_eq!(*tx.inserted_entries(), vec![fork_id(0, 3, 4).hash].into_iter().collect());
assert!(tx.removed_entries().is_empty());
assert_eq!(*tx.updated_meta(), Some(Metadata { finalized: Some(correct_id(2)), unfinalized: vec![correct_id(4), fork_id(0, 3, 4)] }));
@@ -976,9 +1139,13 @@ pub mod tests {
.with_meta(Some(correct_id(2)), vec![])
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), nfin).unwrap(), None);
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), nfin, &Default::default())
.unwrap(),
None,
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
@@ -989,19 +1156,23 @@ pub mod tests {
.with_meta(Some(correct_id(2)), vec![])
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), nfin).unwrap(),
Some(CommitOperation::AddNewFork(Entry { valid_from: correct_id(3), value: 3 })));
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), nfin, &Default::default())
.unwrap(),
Some(CommitOperation::AddNewFork(Entry { valid_from: correct_id(3), value: 3 })),
);
assert_eq!(*tx.inserted_entries(), vec![correct_id(3).hash].into_iter().collect());
assert!(tx.removed_entries().is_empty());
assert_eq!(*tx.updated_meta(), Some(Metadata { finalized: Some(correct_id(2)), unfinalized: vec![correct_id(3)] }));
// when inserting finalized entry AND there are no previous finalized entries
let cache = ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), correct_id(2));
let cache = ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), correct_id(2)).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(
cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), fin).unwrap(),
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), fin, &Default::default())
.unwrap(),
Some(CommitOperation::BlockFinalized(
correct_id(3),
Some(Entry { valid_from: correct_id(3), value: 3 }),
@@ -1017,17 +1188,19 @@ pub mod tests {
.with_meta(Some(correct_id(2)), vec![])
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())));
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin, &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())),
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
// when inserting finalized entry AND value differs from previous finalized
let mut tx = DummyTransaction::new();
assert_eq!(
cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), fin).unwrap(),
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), fin, &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(
correct_id(3),
Some(Entry { valid_from: correct_id(3), value: 3 }),
@@ -1045,10 +1218,12 @@ pub mod tests {
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
.with_entry(fork_id(0, 1, 3), StorageEntry { prev_valid_from: None, value: 13 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, vec![0].into_iter().collect())));
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin, &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, vec![0].into_iter().collect())),
);
}
#[test]
@@ -1060,13 +1235,18 @@ pub mod tests {
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_finalize(&mut tx, correct_id(2), correct_id(3)).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())));
assert_eq!(
cache.do_on_block_finalize(&mut tx, correct_id(2), correct_id(3), &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())),
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
assert_eq!(
*tx.updated_meta(),
Some(Metadata { finalized: Some(correct_id(2)), unfinalized: vec![correct_id(5)] }),
);
// finalization finalizes entry
let cache = ListCache::new(
DummyStorage::new()
@@ -1074,10 +1254,10 @@ pub mod tests {
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 }),
PruningStrategy::ByDepth(1024), correct_id(4)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(
cache.on_block_finalize(&mut tx, correct_id(4), correct_id(5)).unwrap(),
cache.do_on_block_finalize(&mut tx, correct_id(4), correct_id(5), &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(
correct_id(5),
Some(Entry { valid_from: correct_id(5), value: 5 }),
@@ -1094,10 +1274,12 @@ pub mod tests {
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
.with_entry(fork_id(0, 1, 3), StorageEntry { prev_valid_from: None, value: 13 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_finalize(&mut tx, correct_id(2), correct_id(3)).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, vec![0].into_iter().collect())));
assert_eq!(
cache.do_on_block_finalize(&mut tx, correct_id(2), correct_id(3), &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, vec![0].into_iter().collect())),
);
}
#[test]
@@ -1109,25 +1291,29 @@ pub mod tests {
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 })
.with_entry(correct_id(6), StorageEntry { prev_valid_from: Some(correct_id(5)), value: 6 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
// when new block is appended to unfinalized fork
cache.on_transaction_commit(CommitOperation::AppendNewBlock(0, correct_id(6)));
cache.on_transaction_commit(vec![CommitOperation::AppendNewBlock(0, correct_id(6))].into());
assert_eq!(cache.unfinalized[0].best_block, Some(correct_id(6)));
// when new entry is appended to unfinalized fork
cache.on_transaction_commit(CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(7), value: 7 }));
cache.on_transaction_commit(vec![
CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(7), value: 7 }),
].into());
assert_eq!(cache.unfinalized[0].best_block, Some(correct_id(7)));
assert_eq!(cache.unfinalized[0].head, Entry { valid_from: correct_id(7), value: 7 });
// when new fork is added
cache.on_transaction_commit(CommitOperation::AddNewFork(Entry { valid_from: correct_id(10), value: 10 }));
cache.on_transaction_commit(vec![
CommitOperation::AddNewFork(Entry { valid_from: correct_id(10), value: 10 }),
].into());
assert_eq!(cache.unfinalized[2].best_block, Some(correct_id(10)));
assert_eq!(cache.unfinalized[2].head, Entry { valid_from: correct_id(10), value: 10 });
// when block is finalized + entry is finalized + unfinalized forks are deleted
cache.on_transaction_commit(CommitOperation::BlockFinalized(
cache.on_transaction_commit(vec![CommitOperation::BlockFinalized(
correct_id(20),
Some(Entry { valid_from: correct_id(20), value: 20 }),
vec![0, 1, 2].into_iter().collect(),
));
)].into());
assert_eq!(cache.best_finalized_block, correct_id(20));
assert_eq!(cache.best_finalized_entry, Some(Entry { valid_from: correct_id(20), value: 20 }));
assert!(cache.unfinalized.is_empty());
@@ -1148,7 +1334,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(test_header(5)),
PruningStrategy::ByDepth(1024), correct_id(0)
).find_unfinalized_fork(&correct_id(4)).unwrap().unwrap().head.valid_from, correct_id(5));
).unwrap().find_unfinalized_fork((&correct_id(4)).into()).unwrap().unwrap().head.valid_from, correct_id(5));
// --- [2] ---------------> [5]
// ----------> [3] ---> 4
assert_eq!(ListCache::new(
@@ -1165,7 +1351,8 @@ pub mod tests {
.with_header(fork_header(0, 1, 3))
.with_header(fork_header(0, 1, 4)),
PruningStrategy::ByDepth(1024), correct_id(0)
).find_unfinalized_fork(&fork_id(0, 1, 4)).unwrap().unwrap().head.valid_from, fork_id(0, 1, 3));
).unwrap()
.find_unfinalized_fork((&fork_id(0, 1, 4)).into()).unwrap().unwrap().head.valid_from, fork_id(0, 1, 3));
// --- [2] ---------------> [5]
// ----------> [3]
// -----------------> 4
@@ -1185,7 +1372,7 @@ pub mod tests {
.with_header(fork_header(1, 1, 3))
.with_header(fork_header(1, 1, 4)),
PruningStrategy::ByDepth(1024), correct_id(0)
).find_unfinalized_fork(&fork_id(1, 1, 4)).unwrap().is_none());
).unwrap().find_unfinalized_fork((&fork_id(1, 1, 4)).into()).unwrap().is_none());
}
#[test]
@@ -1195,7 +1382,7 @@ pub mod tests {
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 })
.with_entry(test_id(50), StorageEntry { prev_valid_from: None, value: 50 });
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: test_id(100), value: 0 } }
.matches(&storage, &test_id(20)).unwrap(), false);
.matches(&storage, (&test_id(20)).into()).unwrap(), false);
// when block is not connected to the begin block
let storage = DummyStorage::new()
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 })
@@ -1206,7 +1393,7 @@ pub mod tests {
.with_header(fork_header(0, 2, 4))
.with_header(fork_header(0, 2, 3));
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: correct_id(5), value: 100 } }
.matches(&storage, &fork_id(0, 2, 4)).unwrap(), false);
.matches(&storage, (&fork_id(0, 2, 4)).into()).unwrap(), false);
// when block is not connected to the end block
let storage = DummyStorage::new()
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 })
@@ -1216,14 +1403,14 @@ pub mod tests {
.with_header(test_header(3))
.with_header(fork_header(0, 3, 4));
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: correct_id(5), value: 100 } }
.matches(&storage, &fork_id(0, 3, 4)).unwrap(), false);
.matches(&storage, (&fork_id(0, 3, 4)).into()).unwrap(), false);
// when block is connected to the begin block AND end is open
let storage = DummyStorage::new()
.with_entry(correct_id(5), StorageEntry { prev_valid_from: None, value: 100 })
.with_header(test_header(5))
.with_header(test_header(6));
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: correct_id(5), value: 100 } }
.matches(&storage, &correct_id(6)).unwrap(), true);
.matches(&storage, (&correct_id(6)).into()).unwrap(), true);
// when block is connected to the begin block AND to the end block
let storage = DummyStorage::new()
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 })
@@ -1232,7 +1419,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(test_header(3));
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: correct_id(5), value: 100 } }
.matches(&storage, &correct_id(4)).unwrap(), true);
.matches(&storage, (&correct_id(4)).into()).unwrap(), true);
}
#[test]
@@ -1338,9 +1525,21 @@ pub mod tests {
#[test]
fn is_connected_to_block_fails() {
// when storage returns error
assert!(chain::is_connected_to_block::<_, u64, _>(&FaultyStorage, &test_id(1), &test_id(100)).is_err());
assert!(
chain::is_connected_to_block::<_, u64, _>(
&FaultyStorage,
(&test_id(1)).into(),
&test_id(100),
).is_err(),
);
// when there's no header in the storage
assert!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new(), &test_id(1), &test_id(100)).is_err());
assert!(
chain::is_connected_to_block::<_, u64, _>(
&DummyStorage::new(),
(&test_id(1)).into(),
&test_id(100),
).is_err(),
);
}
#[test]
@@ -1348,35 +1547,35 @@ pub mod tests {
// when without iterations we end up with different block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(1)),
&test_id(1), &correct_id(1)).unwrap(), false);
(&test_id(1)).into(), &correct_id(1)).unwrap(), false);
// when with ASC iterations we end up with different block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(0))
.with_header(test_header(1))
.with_header(test_header(2)),
&test_id(0), &correct_id(2)).unwrap(), false);
(&test_id(0)).into(), &correct_id(2)).unwrap(), false);
// when with DESC iterations we end up with different block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(0))
.with_header(test_header(1))
.with_header(test_header(2)),
&correct_id(2), &test_id(0)).unwrap(), false);
(&correct_id(2)).into(), &test_id(0)).unwrap(), false);
// when without iterations we end up with the same block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(1)),
&correct_id(1), &correct_id(1)).unwrap(), true);
(&correct_id(1)).into(), &correct_id(1)).unwrap(), true);
// when with ASC iterations we end up with the same block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(0))
.with_header(test_header(1))
.with_header(test_header(2)),
&correct_id(0), &correct_id(2)).unwrap(), true);
(&correct_id(0)).into(), &correct_id(2)).unwrap(), true);
// when with DESC iterations we end up with the same block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(0))
.with_header(test_header(1))
.with_header(test_header(2)),
&correct_id(2), &correct_id(0)).unwrap(), true);
(&correct_id(2)).into(), &correct_id(0)).unwrap(), true);
}
#[test]
@@ -1454,7 +1653,7 @@ pub mod tests {
.with_entry(test_id(10), StorageEntry { prev_valid_from: None, value: 10 })
.with_entry(test_id(20), StorageEntry { prev_valid_from: Some(test_id(10)), value: 20 })
.with_entry(test_id(30), StorageEntry { prev_valid_from: Some(test_id(20)), value: 30 }),
strategy, test_id(9));
strategy, test_id(9)).unwrap();
let mut tx = DummyTransaction::new();
// when finalizing entry #10: no entries pruned
@@ -1515,28 +1714,64 @@ pub mod tests {
.with_header(fork_header(1, 2, 5))
.with_header(fork_header(2, 4, 5)),
PruningStrategy::ByDepth(1024), correct_id(1)
);
).unwrap();
// when 5 is reverted: entry 5 is truncated
let op = cache.on_block_revert(&mut DummyTransaction::new(), &correct_id(5)).unwrap();
let op = cache.do_on_block_revert(&mut DummyTransaction::new(), &correct_id(5)).unwrap();
assert_eq!(op, CommitOperation::BlockReverted(vec![
(0, Some(Fork { best_block: None, head: Entry { valid_from: correct_id(4), value: 4 } })),
].into_iter().collect()));
cache.on_transaction_commit(op);
cache.on_transaction_commit(vec![op].into());
// when 3 is reverted: entries 4+5' are truncated
let op = cache.on_block_revert(&mut DummyTransaction::new(), &correct_id(3)).unwrap();
let op = cache.do_on_block_revert(&mut DummyTransaction::new(), &correct_id(3)).unwrap();
assert_eq!(op, CommitOperation::BlockReverted(vec![
(0, None),
(2, None),
].into_iter().collect()));
cache.on_transaction_commit(op);
cache.on_transaction_commit(vec![op].into());
// when 2 is reverted: entries 4'+5' are truncated
let op = cache.on_block_revert(&mut DummyTransaction::new(), &correct_id(2)).unwrap();
let op = cache.do_on_block_revert(&mut DummyTransaction::new(), &correct_id(2)).unwrap();
assert_eq!(op, CommitOperation::BlockReverted(vec![
(0, None),
].into_iter().collect()));
cache.on_transaction_commit(op);
cache.on_transaction_commit(vec![op].into());
}
#[test]
fn append_commit_operation_works() {
let mut ops = CommitOperations::default();
ops.append(None);
assert_eq!(ops.operations, Vec::new());
ops.append(Some(CommitOperation::BlockFinalized(
test_id(10),
Some(Entry { valid_from: test_id(10), value: 10 }),
vec![5].into_iter().collect(),
)));
assert_eq!(
ops.operations,
vec![CommitOperation::BlockFinalized(
test_id(10),
Some(Entry { valid_from: test_id(10), value: 10 }),
vec![5].into_iter().collect(),
)],
);
ops.append(Some(CommitOperation::BlockFinalized(
test_id(20),
Some(Entry { valid_from: test_id(20), value: 20 }),
vec![5, 6].into_iter().collect(),
)));
assert_eq!(
ops.operations,
vec![CommitOperation::BlockFinalized(
test_id(20),
Some(Entry { valid_from: test_id(20), value: 20 }),
vec![5, 6].into_iter().collect(),
)],
);
}
}
substrate/client/db/src/cache/list_storage.rs
Vendored
+3 -1
View File
@@ -222,7 +222,9 @@ mod meta {
unfinalized.push(&entry.valid_from);
},
CommitOperation::BlockFinalized(_, ref finalizing_entry, ref forks) => {
finalized = finalizing_entry.as_ref().map(|entry| &entry.valid_from);
if let Some(finalizing_entry) = finalizing_entry.as_ref() {
finalized = Some(&finalizing_entry.valid_from);
}
for fork_index in forks.iter().rev() {
unfinalized.remove(*fork_index);
}
substrate/client/db/src/cache/mod.rs
Vendored
+88 -62
View File
@@ -16,7 +16,7 @@
//! DB-backed cache of blockchain data.
use std::{sync::Arc, collections::HashMap};
use std::{sync::Arc, collections::{HashMap, hash_map::Entry}};
use parking_lot::RwLock;
use kvdb::{KeyValueDB, DBTransaction};
@@ -51,8 +51,10 @@ pub enum EntryType {
/// Block identifier that holds both hash and number.
#[derive(Clone, Debug, Encode, Decode, PartialEq)]
pub struct ComplexBlockId<Block: BlockT> {
hash: Block::Hash,
number: NumberFor<Block>,
/// Hash of the block.
pub(crate) hash: Block::Hash,
/// Number of the block.
pub(crate) number: NumberFor<Block>,
}
impl<Block: BlockT> ComplexBlockId<Block> {
@@ -79,7 +81,7 @@ pub struct DbCache<Block: BlockT> {
db: Arc<dyn KeyValueDB>,
key_lookup_column: u32,
header_column: u32,
authorities_column: u32,
cache_column: u32,
genesis_hash: Block::Hash,
best_finalized_block: ComplexBlockId<Block>,
}
@@ -90,7 +92,7 @@ impl<Block: BlockT> DbCache<Block> {
db: Arc<dyn KeyValueDB>,
key_lookup_column: u32,
header_column: u32,
authorities_column: u32,
cache_column: u32,
genesis_hash: Block::Hash,
best_finalized_block: ComplexBlockId<Block>,
) -> Self {
@@ -99,7 +101,7 @@ impl<Block: BlockT> DbCache<Block> {
db,
key_lookup_column,
header_column,
authorities_column,
cache_column,
genesis_hash,
best_finalized_block,
}
@@ -115,30 +117,48 @@ impl<Block: BlockT> DbCache<Block> {
DbCacheTransaction {
cache: self,
tx,
cache_at_op: HashMap::new(),
cache_at_ops: HashMap::new(),
best_finalized_block: None,
}
}
/// Begin cache transaction with given ops.
pub fn transaction_with_ops<'a>(
&'a mut self,
tx: &'a mut DBTransaction,
ops: DbCacheTransactionOps<Block>,
) -> DbCacheTransaction<'a, Block> {
DbCacheTransaction {
cache: self,
tx,
cache_at_ops: ops.cache_at_ops,
best_finalized_block: ops.best_finalized_block,
}
}
/// Run post-commit cache operations.
pub fn commit(&mut self, ops: DbCacheTransactionOps<Block>) {
for (name, op) in ops.cache_at_op.into_iter() {
self.get_cache(name).on_transaction_commit(op);
pub fn commit(&mut self, ops: DbCacheTransactionOps<Block>) -> ClientResult<()> {
for (name, ops) in ops.cache_at_ops.into_iter() {
self.get_cache(name)?.on_transaction_commit(ops);
}
if let Some(best_finalized_block) = ops.best_finalized_block {
self.best_finalized_block = best_finalized_block;
}
Ok(())
}
/// Creates `ListCache` with the given name or returns a reference to the existing.
fn get_cache(&mut self, name: CacheKeyId) -> &mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage> {
pub(crate) fn get_cache(
&mut self,
name: CacheKeyId,
) -> ClientResult<&mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage>> {
get_cache_helper(
&mut self.cache_at,
name,
&self.db,
self.key_lookup_column,
self.header_column,
self.authorities_column,
self.cache_column,
&self.best_finalized_block
)
}
@@ -154,34 +174,49 @@ fn get_cache_helper<'a, Block: BlockT>(
header: u32,
cache: u32,
best_finalized_block: &ComplexBlockId<Block>,
) -> &'a mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage> {
cache_at.entry(name).or_insert_with(|| {
ListCache::new(
self::list_storage::DbStorage::new(name.to_vec(), db.clone(),
self::list_storage::DbColumns {
meta: COLUMN_META,
key_lookup,
header,
cache,
},
),
cache_pruning_strategy(name),
best_finalized_block.clone(),
)
})
) -> ClientResult<&'a mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage>> {
match cache_at.entry(name) {
Entry::Occupied(entry) => Ok(entry.into_mut()),
Entry::Vacant(entry) => {
let cache = ListCache::new(
self::list_storage::DbStorage::new(name.to_vec(), db.clone(),
self::list_storage::DbColumns {
meta: COLUMN_META,
key_lookup,
header,
cache,
},
),
cache_pruning_strategy(name),
best_finalized_block.clone(),
)?;
Ok(entry.insert(cache))
}
}
}
/// Cache operations that are to be committed after database transaction is committed.
#[derive(Default)]
pub struct DbCacheTransactionOps<Block: BlockT> {
cache_at_op: HashMap<CacheKeyId, self::list_cache::CommitOperation<Block, Vec<u8>>>,
cache_at_ops: HashMap<CacheKeyId, self::list_cache::CommitOperations<Block, Vec<u8>>>,
best_finalized_block: Option<ComplexBlockId<Block>>,
}
impl<Block: BlockT> DbCacheTransactionOps<Block> {
/// Empty transaction ops.
pub fn empty() -> DbCacheTransactionOps<Block> {
DbCacheTransactionOps {
cache_at_ops: HashMap::new(),
best_finalized_block: None,
}
}
}
/// Database-backed blockchain data cache transaction valid for single block import.
pub struct DbCacheTransaction<'a, Block: BlockT> {
cache: &'a mut DbCache<Block>,
tx: &'a mut DBTransaction,
cache_at_op: HashMap<CacheKeyId, self::list_cache::CommitOperation<Block, Vec<u8>>>,
cache_at_ops: HashMap<CacheKeyId, self::list_cache::CommitOperations<Block, Vec<u8>>>,
best_finalized_block: Option<ComplexBlockId<Block>>,
}
@@ -189,7 +224,7 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
/// Convert transaction into post-commit operations set.
pub fn into_ops(self) -> DbCacheTransactionOps<Block> {
DbCacheTransactionOps {
cache_at_op: self.cache_at_op,
cache_at_ops: self.cache_at_ops,
best_finalized_block: self.best_finalized_block,
}
}
@@ -202,8 +237,6 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
data_at: HashMap<CacheKeyId, Vec<u8>>,
entry_type: EntryType,
) -> ClientResult<Self> {
assert!(self.cache_at_op.is_empty());
// prepare list of caches that are not update
// (we might still need to do some cache maintenance in this case)
let missed_caches = self.cache.cache_at.keys()
@@ -212,8 +245,9 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
.collect::<Vec<_>>();
let mut insert_op = |name: CacheKeyId, value: Option<Vec<u8>>| -> Result<(), sp_blockchain::Error> {
let cache = self.cache.get_cache(name);
let op = cache.on_block_insert(
let cache = self.cache.get_cache(name)?;
let cache_ops = self.cache_at_ops.entry(name).or_default();
cache.on_block_insert(
&mut self::list_storage::DbStorageTransaction::new(
cache.storage(),
&mut self.tx,
@@ -222,10 +256,9 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
block.clone(),
value,
entry_type,
cache_ops,
)?;
if let Some(op) = op {
self.cache_at_op.insert(name, op);
}
Ok(())
};
@@ -245,23 +278,19 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
pub fn on_block_finalize(
mut self,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>
block: ComplexBlockId<Block>,
) -> ClientResult<Self> {
assert!(self.cache_at_op.is_empty());
for (name, cache_at) in self.cache.cache_at.iter() {
let op = cache_at.on_block_finalize(
for (name, cache) in self.cache.cache_at.iter() {
let cache_ops = self.cache_at_ops.entry(*name).or_default();
cache.on_block_finalize(
&mut self::list_storage::DbStorageTransaction::new(
cache_at.storage(),
cache.storage(),
&mut self.tx
),
parent.clone(),
block.clone(),
cache_ops,
)?;
if let Some(op) = op {
self.cache_at_op.insert(name.to_owned(), op);
}
}
self.best_finalized_block = Some(block);
@@ -275,16 +304,15 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
reverted_block: &ComplexBlockId<Block>,
) -> ClientResult<Self> {
for (name, cache) in self.cache.cache_at.iter() {
let op = cache.on_block_revert(
let cache_ops = self.cache_at_ops.entry(*name).or_default();
cache.on_block_revert(
&mut self::list_storage::DbStorageTransaction::new(
cache.storage(),
&mut self.tx
),
reverted_block,
cache_ops,
)?;
assert!(!self.cache_at_op.contains_key(name));
self.cache_at_op.insert(name.to_owned(), op);
}
Ok(self)
@@ -310,7 +338,7 @@ impl<Block: BlockT> BlockchainCache<Block> for DbCacheSync<Block> {
)?;
let tx_ops = tx.into_ops();
db.write(dbtx).map_err(db_err)?;
cache.commit(tx_ops);
cache.commit(tx_ops)?;
Ok(())
}
@@ -318,40 +346,38 @@ impl<Block: BlockT> BlockchainCache<Block> for DbCacheSync<Block> {
&self,
key: &CacheKeyId,
at: &BlockId<Block>,
) -> Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)> {
) -> ClientResult<Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)>> {
let mut cache = self.0.write();
let storage = cache.get_cache(*key).storage();
let cache = cache.get_cache(*key)?;
let storage = cache.storage();
let db = storage.db();
let columns = storage.columns();
let at = match *at {
BlockId::Hash(hash) => {
let header = utils::read_header::<Block>(
let header = utils::require_header::<Block>(
&**db,
columns.key_lookup,
columns.header,
BlockId::Hash(hash.clone())).ok()??;
BlockId::Hash(hash.clone()))?;
ComplexBlockId::new(hash, *header.number())
},
BlockId::Number(number) => {
let hash = utils::read_header::<Block>(
let hash = utils::require_header::<Block>(
&**db,
columns.key_lookup,
columns.header,
BlockId::Number(number.clone())).ok()??.hash();
BlockId::Number(number.clone()))?.hash();
ComplexBlockId::new(hash, number)
},
};
cache.cache_at
.get(key)?
.value_at_block(&at)
cache.value_at_block(&at)
.map(|block_and_value| block_and_value.map(|(begin_block, end_block, value)|
(
(begin_block.number, begin_block.hash),
end_block.map(|end_block| (end_block.number, end_block.hash)),
value,
)))
.ok()?
}
}
substrate/client/db/src/changes_tries_storage.rs
+1015
View File
File diff suppressed because it is too large Load Diff
substrate/client/db/src/lib.rs
+162 -491
View File
File diff suppressed because it is too large Load Diff
substrate/client/db/src/light.rs
+56 -17
View File
@@ -38,7 +38,7 @@ use codec::{Decode, Encode};
use sp_runtime::generic::{DigestItem, BlockId};
use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, One, NumberFor, HasherFor};
use crate::cache::{DbCacheSync, DbCache, ComplexBlockId, EntryType as CacheEntryType};
use crate::utils::{self, meta_keys, Meta, db_err, read_db, block_id_to_lookup_key, read_meta};
use crate::utils::{self, meta_keys, DatabaseType, Meta, db_err, read_db, block_id_to_lookup_key, read_meta};
use crate::{DatabaseSettings, FrozenForDuration};
use log::{trace, warn, debug};
@@ -68,13 +68,10 @@ pub struct LightStorage<Block: BlockT> {
io_stats: FrozenForDuration<kvdb::IoStats>,
}
impl<Block> LightStorage<Block>
where
Block: BlockT,
{
impl<Block: BlockT> LightStorage<Block> {
/// Create new storage with given settings.
pub fn new(config: DatabaseSettings) -> ClientResult<Self> {
let db = crate::utils::open_database(&config, columns::META, "light")?;
let db = crate::utils::open_database::<Block>(&config, DatabaseType::Light)?;
Self::from_kvdb(db as Arc<_>)
}
@@ -89,7 +86,7 @@ impl<Block> LightStorage<Block>
}
fn from_kvdb(db: Arc<dyn KeyValueDB>) -> ClientResult<Self> {
let meta = read_meta::<Block>(&*db, columns::META, columns::HEADER)?;
let meta = read_meta::<Block>(&*db, columns::HEADER)?;
let cache = DbCache::new(
db.clone(),
columns::KEY_LOOKUP,
@@ -417,7 +414,7 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
fn import_header(
&self,
header: Block::Header,
cache_at: HashMap<well_known_cache_keys::Id, Vec<u8>>,
mut cache_at: HashMap<well_known_cache_keys::Id, Vec<u8>>,
leaf_state: NewBlockState,
aux_ops: Vec<(Vec<u8>, Option<Vec<u8>>)>,
) -> ClientResult<()> {
@@ -475,6 +472,13 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
)?;
}
// update changes trie configuration cache
if !cache_at.contains_key(&well_known_cache_keys::CHANGES_TRIE_CONFIG) {
if let Some(new_configuration) = crate::changes_tries_storage::extract_new_configuration(&header) {
cache_at.insert(well_known_cache_keys::CHANGES_TRIE_CONFIG, new_configuration.encode());
}
}
{
let mut cache = self.cache.0.write();
let cache_ops = cache.transaction(&mut transaction)
@@ -487,8 +491,11 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
.into_ops();
debug!("Light DB Commit {:?} ({})", hash, number);
self.db.write(transaction).map_err(db_err)?;
cache.commit(cache_ops);
cache.commit(cache_ops)
.expect("only fails if cache with given name isn't loaded yet;\
cache is already loaded because there are cache_ops; qed");
}
self.update_meta(hash, number, leaf_state.is_best(), finalized);
@@ -543,7 +550,9 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
.into_ops();
self.db.write(transaction).map_err(db_err)?;
cache.commit(cache_ops);
cache.commit(cache_ops)
.expect("only fails if cache with given name isn't loaded yet;\
cache is already loaded because there are cache_ops; qed");
}
self.update_meta(hash, header.number().clone(), false, true);
@@ -603,7 +612,8 @@ fn cht_key<N: TryInto<u32>>(cht_type: u8, block: N) -> ClientResult<[u8; 5]> {
#[cfg(test)]
pub(crate) mod tests {
use sc_client::cht;
use sp_runtime::generic::DigestItem;
use sp_core::ChangesTrieConfiguration;
use sp_runtime::generic::{DigestItem, ChangesTrieSignal};
use sp_runtime::testing::{H256 as Hash, Header, Block as RawBlock, ExtrinsicWrapper};
use sp_blockchain::{lowest_common_ancestor, tree_route};
use super::*;
@@ -964,7 +974,7 @@ pub(crate) mod tests {
}
fn get_authorities(cache: &dyn BlockchainCache<Block>, at: BlockId<Block>) -> Option<Vec<AuthorityId>> {
cache.get_at(&well_known_cache_keys::AUTHORITIES, &at)
cache.get_at(&well_known_cache_keys::AUTHORITIES, &at).unwrap_or(None)
.and_then(|(_, _, val)| Decode::decode(&mut &val[..]).ok())
}
@@ -1148,8 +1158,8 @@ pub(crate) mod tests {
let (genesis_hash, storage) = {
let db = LightStorage::<Block>::new_test();
// before cache is initialized => None
assert_eq!(db.cache().get_at(b"test", &BlockId::Number(0)), None);
// before cache is initialized => Err
assert!(db.cache().get_at(b"test", &BlockId::Number(0)).is_err());
// insert genesis block (no value for cache is provided)
let mut genesis_hash = None;
@@ -1160,14 +1170,14 @@ pub(crate) mod tests {
});
// after genesis is inserted => None
assert_eq!(db.cache().get_at(b"test", &BlockId::Number(0)), None);
assert_eq!(db.cache().get_at(b"test", &BlockId::Number(0)).unwrap(), None);
// initialize cache
db.cache().initialize(b"test", vec![42]).unwrap();
// after genesis is inserted + cache is initialized => Some
assert_eq!(
db.cache().get_at(b"test", &BlockId::Number(0)),
db.cache().get_at(b"test", &BlockId::Number(0)).unwrap(),
Some(((0, genesis_hash.unwrap()), None, vec![42])),
);
@@ -1177,8 +1187,37 @@ pub(crate) mod tests {
// restart && check that after restart value is read from the cache
let db = LightStorage::<Block>::from_kvdb(storage as Arc<_>).expect("failed to create test-db");
assert_eq!(
db.cache().get_at(b"test", &BlockId::Number(0)),
db.cache().get_at(b"test", &BlockId::Number(0)).unwrap(),
Some(((0, genesis_hash.unwrap()), None, vec![42])),
);
}
#[test]
fn changes_trie_configuration_is_tracked_on_light_client() {
let db = LightStorage::<Block>::new_test();
let new_config = Some(ChangesTrieConfiguration::new(2, 2));
// insert block#0 && block#1 (no value for cache is provided)
let hash0 = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
assert_eq!(
db.cache().get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(0)).unwrap()
.map(|(_, _, v)| ChangesTrieConfiguration::decode(&mut &v[..]).unwrap()),
None,
);
// insert configuration at block#1 (starts from block#2)
insert_block(&db, HashMap::new(), || {
let mut header = default_header(&hash0, 1);
header.digest_mut().push(
DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(new_config.clone()))
);
header
});
assert_eq!(
db.cache().get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(1)).unwrap()
.map(|(_, _, v)| Option::<ChangesTrieConfiguration>::decode(&mut &v[..]).unwrap()),
Some(new_config),
);
}
}
substrate/client/db/src/upgrade.rs
+198
View File
@@ -0,0 +1,198 @@
// Copyright 2019-2020 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Database upgrade logic.
use std::fs;
use std::io::{Read, Write, ErrorKind};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use codec::Encode;
use kvdb_rocksdb::{Database, DatabaseConfig};
use parking_lot::RwLock;
use sp_blockchain::{well_known_cache_keys, Cache};
use sp_core::ChangesTrieConfiguration;
use sp_runtime::traits::Block as BlockT;
use crate::{
cache::{ComplexBlockId, DbCache, DbCacheSync},
utils::{DatabaseType, check_database_type, db_err, read_genesis_hash},
};
/// Version file name.
const VERSION_FILE_NAME: &'static str = "db_version";
/// Current db version.
const CURRENT_VERSION: u32 = 1;
/// Number of columns in v0.
const V0_NUM_COLUMNS: u32 = 10;
/// Upgrade database to current version.
pub fn upgrade_db<Block: BlockT>(db_path: &Path, db_type: DatabaseType) -> sp_blockchain::Result<()> {
let db_version = current_version(db_path)?;
match db_version {
0 => migrate_0_to_1::<Block>(db_path, db_type)?,
1 => (),
_ => Err(sp_blockchain::Error::Backend(format!("Future database version: {}", db_version)))?,
}
update_version(db_path)
}
/// Migration from version0 to version1:
/// 1) the number of columns has changed from 10 to 11;
/// 2) changes tries configuration are now cached.
fn migrate_0_to_1<Block: BlockT>(db_path: &Path, db_type: DatabaseType) -> sp_blockchain::Result<()> {
{
let db = open_database(db_path, db_type, V0_NUM_COLUMNS)?;
db.add_column().map_err(db_err)?;
db.flush().map_err(db_err)?;
}
let db = open_database(db_path, db_type, V0_NUM_COLUMNS + 1)?;
const V0_FULL_KEY_LOOKUP_COLUMN: u32 = 3;
const V0_FULL_HEADER_COLUMN: u32 = 4;
const V0_FULL_CACHE_COLUMN: u32 = 10; // that's the column we have just added
const V0_LIGHT_KEY_LOOKUP_COLUMN: u32 = 1;
const V0_LIGHT_HEADER_COLUMN: u32 = 2;
const V0_LIGHT_CACHE_COLUMN: u32 = 3;
let (key_lookup_column, header_column, cache_column) = match db_type {
DatabaseType::Full => (
V0_FULL_KEY_LOOKUP_COLUMN,
V0_FULL_HEADER_COLUMN,
V0_FULL_CACHE_COLUMN,
),
DatabaseType::Light => (
V0_LIGHT_KEY_LOOKUP_COLUMN,
V0_LIGHT_HEADER_COLUMN,
V0_LIGHT_CACHE_COLUMN,
),
};
let genesis_hash: Option<Block::Hash> = read_genesis_hash(&db)?;
if let Some(genesis_hash) = genesis_hash {
let cache: DbCacheSync<Block> = DbCacheSync(RwLock::new(DbCache::new(
Arc::new(db),
key_lookup_column,
header_column,
cache_column,
genesis_hash,
ComplexBlockId::new(genesis_hash, 0.into()),
)));
let changes_trie_config: Option<ChangesTrieConfiguration> = None;
cache.initialize(&well_known_cache_keys::CHANGES_TRIE_CONFIG, changes_trie_config.encode())?;
}
Ok(())
}
/// Reads current database version from the file at given path.
/// If the file does not exist returns 0.
fn current_version(path: &Path) -> sp_blockchain::Result<u32> {
let unknown_version_err = || sp_blockchain::Error::Backend("Unknown database version".into());
match fs::File::open(version_file_path(path)) {
Err(ref err) if err.kind() == ErrorKind::NotFound => Ok(0),
Err(_) => Err(unknown_version_err()),
Ok(mut file) => {
let mut s = String::new();
file.read_to_string(&mut s).map_err(|_| unknown_version_err())?;
u32::from_str_radix(&s, 10).map_err(|_| unknown_version_err())
},
}
}
/// Opens database of givent type with given number of columns.
fn open_database(db_path: &Path, db_type: DatabaseType, db_columns: u32) -> sp_blockchain::Result<Database> {
let db_path = db_path.to_str()
.ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?;
let db_cfg = DatabaseConfig::with_columns(db_columns);
let db = Database::open(&db_cfg, db_path).map_err(db_err)?;
check_database_type(&db, db_type)?;
Ok(db)
}
/// Writes current database version to the file.
/// Creates a new file if the version file does not exist yet.
fn update_version(path: &Path) -> sp_blockchain::Result<()> {
fs::create_dir_all(path).map_err(db_err)?;
let mut file = fs::File::create(version_file_path(path)).map_err(db_err)?;
file.write_all(format!("{}", CURRENT_VERSION).as_bytes()).map_err(db_err)?;
Ok(())
}
/// Returns the version file path.
fn version_file_path(path: &Path) -> PathBuf {
let mut file_path = path.to_owned();
file_path.push(VERSION_FILE_NAME);
file_path
}
#[cfg(test)]
mod tests {
use sc_state_db::PruningMode;
use crate::{DatabaseSettings, DatabaseSettingsSrc};
use crate::tests::Block;
use super::*;
fn create_db(db_path: &Path, version: Option<u32>) {
let db_cfg = DatabaseConfig::with_columns(V0_NUM_COLUMNS);
Database::open(&db_cfg, db_path.to_str().unwrap()).unwrap();
if let Some(version) = version {
fs::create_dir_all(db_path).unwrap();
let mut file = fs::File::create(version_file_path(db_path)).unwrap();
file.write_all(format!("{}", version).as_bytes()).unwrap();
}
}
fn open_database(db_path: &Path) -> sp_blockchain::Result<()> {
crate::utils::open_database::<Block>(&DatabaseSettings {
state_cache_size: 0,
state_cache_child_ratio: None,
pruning: PruningMode::ArchiveAll,
source: DatabaseSettingsSrc::Path { path: db_path.to_owned(), cache_size: None },
}, DatabaseType::Full).map(|_| ())
}
#[test]
fn downgrade_never_happens() {
let db_dir = tempdir::TempDir::new("").unwrap();
create_db(db_dir.path(), Some(CURRENT_VERSION + 1));
assert!(open_database(db_dir.path()).is_err());
}
#[test]
fn open_empty_database_works() {
let db_dir = tempdir::TempDir::new("").unwrap();
open_database(db_dir.path()).unwrap();
open_database(db_dir.path()).unwrap();
assert_eq!(current_version(db_dir.path()).unwrap(), CURRENT_VERSION);
}
#[test]
fn upgrade_from_0_to_1_works() {
for version_from_file in &[None, Some(0)] {
let db_dir = tempdir::TempDir::new("").unwrap();
let db_path = db_dir.path();
create_db(db_path, *version_from_file);
open_database(db_path).unwrap();
assert_eq!(current_version(db_path).unwrap(), CURRENT_VERSION);
}
}
}
substrate/client/db/src/utils.rs
+66 -22
View File
@@ -21,7 +21,7 @@ use std::sync::Arc;
use std::{io, convert::TryInto};
use kvdb::{KeyValueDB, DBTransaction};
#[cfg(feature = "kvdb-rocksdb")]
#[cfg(any(feature = "kvdb-rocksdb", test))]
use kvdb_rocksdb::{Database, DatabaseConfig};
use log::debug;
@@ -36,7 +36,7 @@ use crate::{DatabaseSettings, DatabaseSettingsSrc};
/// Number of columns in the db. Must be the same for both full && light dbs.
/// Otherwise RocksDb will fail to open database && check its type.
pub const NUM_COLUMNS: u32 = 10;
pub const NUM_COLUMNS: u32 = 11;
/// Meta column. The set of keys in the column is shared by full && light storages.
pub const COLUMN_META: u32 = 0;
@@ -50,6 +50,8 @@ pub mod meta_keys {
pub const FINALIZED_BLOCK: &[u8; 5] = b"final";
/// Meta information prefix for list-based caches.
pub const CACHE_META_PREFIX: &[u8; 5] = b"cache";
/// Meta information for changes tries key.
pub const CHANGES_TRIES_META: &[u8; 5] = b"ctrie";
/// Genesis block hash.
pub const GENESIS_HASH: &[u8; 3] = b"gen";
/// Leaves prefix list key.
@@ -76,6 +78,15 @@ pub struct Meta<N, H> {
/// A block lookup key: used for canonical lookup from block number to hash
pub type NumberIndexKey = [u8; 4];
/// Database type.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum DatabaseType {
/// Full node database.
Full,
/// Light node database.
Light,
}
/// Convert block number into short lookup key (LE representation) for
/// blocks that are in the canonical chain.
///
@@ -203,14 +214,17 @@ pub fn db_err(err: io::Error) -> sp_blockchain::Error {
}
/// Open RocksDB database.
pub fn open_database(
pub fn open_database<Block: BlockT>(
config: &DatabaseSettings,
col_meta: u32,
db_type: &str
db_type: DatabaseType,
) -> sp_blockchain::Result<Arc<dyn KeyValueDB>> {
let db: Arc<dyn KeyValueDB> = match &config.source {
#[cfg(feature = "kvdb-rocksdb")]
#[cfg(any(feature = "kvdb-rocksdb", test))]
DatabaseSettingsSrc::Path { path, cache_size } => {
// first upgrade database to required version
crate::upgrade::upgrade_db::<Block>(&path, db_type)?;
// and now open database assuming that it has the latest version
let mut db_config = DatabaseConfig::with_columns(NUM_COLUMNS);
if let Some(cache_size) = cache_size {
@@ -232,7 +246,7 @@ pub fn open_database(
.ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?;
Arc::new(Database::open(&db_config, &path).map_err(db_err)?)
},
#[cfg(not(feature = "kvdb-rocksdb"))]
#[cfg(not(any(feature = "kvdb-rocksdb", test)))]
DatabaseSettingsSrc::Path { .. } => {
let msg = "Try to open RocksDB database with RocksDB disabled".into();
return Err(sp_blockchain::Error::Backend(msg));
@@ -240,22 +254,28 @@ pub fn open_database(
DatabaseSettingsSrc::Custom(db) => db.clone(),
};
// check database type
match db.get(col_meta, meta_keys::TYPE).map_err(db_err)? {
check_database_type(&*db, db_type)?;
Ok(db)
}
/// Check database type.
pub fn check_database_type(db: &dyn KeyValueDB, db_type: DatabaseType) -> sp_blockchain::Result<()> {
match db.get(COLUMN_META, meta_keys::TYPE).map_err(db_err)? {
Some(stored_type) => {
if db_type.as_bytes() != &*stored_type {
if db_type.as_str().as_bytes() != &*stored_type {
return Err(sp_blockchain::Error::Backend(
format!("Unexpected database type. Expected: {}", db_type)).into());
format!("Unexpected database type. Expected: {}", db_type.as_str())).into());
}
},
None => {
let mut transaction = DBTransaction::new();
transaction.put(col_meta, meta_keys::TYPE, db_type.as_bytes());
transaction.put(COLUMN_META, meta_keys::TYPE, db_type.as_str().as_bytes());
db.write(transaction).map_err(db_err)?;
},
}
Ok(db)
Ok(())
}
/// Read database column entry for the given block.
@@ -304,20 +324,15 @@ pub fn require_header<Block: BlockT>(
}
/// Read meta from the database.
pub fn read_meta<Block>(db: &dyn KeyValueDB, col_meta: u32, col_header: u32) -> Result<
pub fn read_meta<Block>(db: &dyn KeyValueDB, col_header: u32) -> Result<
Meta<<<Block as BlockT>::Header as HeaderT>::Number, Block::Hash>,
sp_blockchain::Error,
>
where
Block: BlockT,
{
let genesis_hash: Block::Hash = match db.get(col_meta, meta_keys::GENESIS_HASH).map_err(db_err)? {
Some(h) => match Decode::decode(&mut &h[..]) {
Ok(h) => h,
Err(err) => return Err(sp_blockchain::Error::Backend(
format!("Error decoding genesis hash: {}", err)
)),
},
let genesis_hash: Block::Hash = match read_genesis_hash(db)? {
Some(genesis_hash) => genesis_hash,
None => return Ok(Meta {
best_hash: Default::default(),
best_number: Zero::zero(),
@@ -328,7 +343,7 @@ pub fn read_meta<Block>(db: &dyn KeyValueDB, col_meta: u32, col_header: u32) ->
};
let load_meta_block = |desc, key| -> Result<_, sp_blockchain::Error> {
if let Some(Some(header)) = db.get(col_meta, key).and_then(|id|
if let Some(Some(header)) = db.get(COLUMN_META, key).and_then(|id|
match id {
Some(id) => db.get(col_header, &id).map(|h| h.map(|b| Block::Header::decode(&mut &b[..]).ok())),
None => Ok(None),
@@ -354,6 +369,29 @@ pub fn read_meta<Block>(db: &dyn KeyValueDB, col_meta: u32, col_header: u32) ->
})
}
/// Read genesis hash from database.
pub fn read_genesis_hash<Hash: Decode>(db: &dyn KeyValueDB) -> sp_blockchain::Result<Option<Hash>> {
match db.get(COLUMN_META, meta_keys::GENESIS_HASH).map_err(db_err)? {
Some(h) => match Decode::decode(&mut &h[..]) {
Ok(h) => Ok(Some(h)),
Err(err) => Err(sp_blockchain::Error::Backend(
format!("Error decoding genesis hash: {}", err)
)),
},
None => Ok(None),
}
}
impl DatabaseType {
/// Returns str representation of the type.
pub fn as_str(&self) -> &'static str {
match *self {
DatabaseType::Full => "full",
DatabaseType::Light => "light",
}
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -368,4 +406,10 @@ mod tests {
_ => unreachable!(),
};
}
#[test]
fn database_type_as_str_works() {
assert_eq!(DatabaseType::Full.as_str(), "full");
assert_eq!(DatabaseType::Light.as_str(), "light");
}
}
substrate/client/finality-grandpa/src/tests.rs
+2 -4
View File
@@ -287,12 +287,10 @@ impl ApiExt<Block> for RuntimeApi {
unimplemented!("Not required for testing!")
}
fn into_storage_changes<
T: sp_api::ChangesTrieStorage<sp_api::HasherFor<Block>, sp_api::NumberFor<Block>>
>(
fn into_storage_changes(
&self,
_: &Self::StateBackend,
_: Option<&T>,
_: Option<&sp_api::ChangesTrieState<sp_api::HasherFor<Block>, sp_api::NumberFor<Block>>>,
_: <Block as sp_api::BlockT>::Hash,
) -> std::result::Result<sp_api::StorageChanges<Self::StateBackend, Block>, String>
where Self: Sized
substrate/client/network/src/protocol/light_dispatch.rs
+6 -2
View File
@@ -691,7 +691,7 @@ pub mod tests {
use crate::message::{self, BlockAttributes, Direction, FromBlock, RequestId};
use libp2p::PeerId;
use super::{REQUEST_TIMEOUT, LightDispatch, LightDispatchNetwork, RequestData, StorageProof};
use sp_test_primitives::{changes_trie_config, Block, Extrinsic, Header};
use sp_test_primitives::{Block, Extrinsic, Header};
struct DummyFetchChecker { ok: bool }
@@ -1095,7 +1095,11 @@ pub mod tests {
let (tx, response) = oneshot::channel();
light_dispatch.add_request(&mut network_interface, RequestData::RemoteChanges(RemoteChangesRequest {
changes_trie_config: changes_trie_config(),
changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange {
zero: (0, Default::default()),
end: None,
config: Some(sp_core::ChangesTrieConfiguration::new(4, 2)),
}],
first_block: (1, Default::default()),
last_block: (100, Default::default()),
max_block: (100, Default::default()),
substrate/client/rpc/src/state/tests.rs
+4 -2
View File
@@ -21,7 +21,7 @@ use self::error::Error;
use std::sync::Arc;
use assert_matches::assert_matches;
use futures01::stream::Stream;
use sp_core::storage::{well_known_keys, ChildInfo};
use sp_core::{storage::{well_known_keys, ChildInfo}, ChangesTrieConfiguration};
use sp_core::hash::H256;
use sp_io::hashing::blake2_256;
use substrate_test_runtime_client::{
@@ -378,7 +378,9 @@ fn should_query_storage() {
}
run_tests(Arc::new(substrate_test_runtime_client::new()));
run_tests(Arc::new(TestClientBuilder::new().set_support_changes_trie(true).build()));
run_tests(Arc::new(TestClientBuilder::new()
.changes_trie_config(Some(ChangesTrieConfiguration::new(4, 2)))
.build()));
}
#[test]
substrate/client/src/call_executor.rs
+8 -7
View File
@@ -17,7 +17,7 @@
use std::{sync::Arc, panic::UnwindSafe, result, cell::RefCell};
use codec::{Encode, Decode};
use sp_runtime::{
generic::BlockId, traits::{Block as BlockT, HasherFor},
generic::BlockId, traits::{Block as BlockT, HasherFor, NumberFor},
};
use sp_state_machine::{
self, OverlayedChanges, Ext, ExecutionManager, StateMachine, ExecutionStrategy,
@@ -80,7 +80,7 @@ where
let state = self.backend.state_at(*id)?;
let return_data = StateMachine::new(
&state,
self.backend.changes_trie_storage(),
backend::changes_tries_state_at_block(id, self.backend.changes_trie_storage())?,
&mut changes,
&self.executor,
method,
@@ -132,6 +132,7 @@ where
}
let mut state = self.backend.state_at(*at)?;
let changes_trie_state = backend::changes_tries_state_at_block(at, self.backend.changes_trie_storage())?;
let mut storage_transaction_cache = storage_transaction_cache.map(|c| c.borrow_mut());
@@ -150,7 +151,7 @@ where
StateMachine::new(
&backend,
self.backend.changes_trie_storage(),
changes_trie_state,
&mut *changes.borrow_mut(),
&self.executor,
method,
@@ -163,7 +164,7 @@ where
}
None => StateMachine::new(
&state,
self.backend.changes_trie_storage(),
changes_trie_state,
&mut *changes.borrow_mut(),
&self.executor,
method,
@@ -183,13 +184,13 @@ where
fn runtime_version(&self, id: &BlockId<Block>) -> sp_blockchain::Result<RuntimeVersion> {
let mut overlay = OverlayedChanges::default();
let state = self.backend.state_at(*id)?;
let changes_trie_state = backend::changes_tries_state_at_block(id, self.backend.changes_trie_storage())?;
let mut cache = StorageTransactionCache::<Block, B::State>::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
&state,
self.backend.changes_trie_storage(),
changes_trie_state,
None,
);
let version = self.executor.runtime_version(&mut ext);
@@ -207,7 +208,7 @@ where
method: &str,
call_data: &[u8]
) -> Result<(Vec<u8>, StorageProof), sp_blockchain::Error> {
sp_state_machine::prove_execution_on_trie_backend(
sp_state_machine::prove_execution_on_trie_backend::<_, _, NumberFor<Block>, _>(
trie_state,
overlay,
&self.executor,
substrate/client/src/client.rs
+230 -76
View File
@@ -66,6 +66,7 @@ pub use sc_client_api::{
backend::{
self, BlockImportOperation, PrunableStateChangesTrieStorage,
ClientImportOperation, Finalizer, ImportSummary, NewBlockState,
changes_tries_state_at_block,
},
client::{
ImportNotifications, FinalityNotification, FinalityNotifications, BlockImportNotification,
@@ -407,18 +408,26 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
first: NumberFor<Block>,
last: BlockId<Block>,
) -> sp_blockchain::Result<Option<(NumberFor<Block>, BlockId<Block>)>> {
let (config, storage) = match self.require_changes_trie().ok() {
Some((config, storage)) => (config, storage),
None => return Ok(None),
};
let last_num = self.backend.blockchain().expect_block_number_from_id(&last)?;
if first > last_num {
let last_number = self.backend.blockchain().expect_block_number_from_id(&last)?;
let last_hash = self.backend.blockchain().expect_block_hash_from_id(&last)?;
if first > last_number {
return Err(sp_blockchain::Error::ChangesTrieAccessFailed("Invalid changes trie range".into()));
}
let finalized_number = self.backend.blockchain().info().finalized_number;
let oldest = storage.oldest_changes_trie_block(&config, finalized_number);
let first = ::std::cmp::max(first, oldest);
Ok(Some((first, last)))
let (storage, configs) = match self.require_changes_trie(first, last_hash, false).ok() {
Some((storage, configs)) => (storage, configs),
None => return Ok(None),
};
let first_available_changes_trie = configs.last().map(|config| config.0);
match first_available_changes_trie {
Some(first_available_changes_trie) => {
let oldest_unpruned = storage.oldest_pruned_digest_range_end();
let first = std::cmp::max(first_available_changes_trie, oldest_unpruned);
Ok(Some((first, last)))
},
None => Ok(None)
}
}
/// Get pairs of (block, extrinsic) where key has been changed at given blocks range.
@@ -432,30 +441,42 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
storage_key: Option<&StorageKey>,
key: &StorageKey
) -> sp_blockchain::Result<Vec<(NumberFor<Block>, u32)>> {
let (config, storage) = self.require_changes_trie()?;
let last_number = self.backend.blockchain().expect_block_number_from_id(&last)?;
let last_hash = self.backend.blockchain().expect_block_hash_from_id(&last)?;
let (storage, configs) = self.require_changes_trie(first, last_hash, true)?;
// FIXME: remove this in https://github.com/paritytech/substrate/pull/3201
let config_range = ChangesTrieConfigurationRange {
config: &config,
zero: Zero::zero(),
end: None,
};
let mut result = Vec::new();
let best_number = self.backend.blockchain().info().best_number;
for (config_zero, config_end, config) in configs {
let range_first = ::std::cmp::max(first, config_zero + One::one());
let range_anchor = match config_end {
Some((config_end_number, config_end_hash)) => if last_number > config_end_number {
ChangesTrieAnchorBlockId { hash: config_end_hash, number: config_end_number }
} else {
ChangesTrieAnchorBlockId { hash: convert_hash(&last_hash), number: last_number }
},
None => ChangesTrieAnchorBlockId { hash: convert_hash(&last_hash), number: last_number },
};
key_changes::<HasherFor<Block>, _>(
config_range,
&*storage,
first,
&ChangesTrieAnchorBlockId {
hash: convert_hash(&last_hash),
number: last_number,
},
self.backend.blockchain().info().best_number,
storage_key.as_ref().map(|sk| sk.0.as_slice()),
&key.0)
.and_then(|r| r.map(|r| r.map(|(block, tx)| (block, tx))).collect::<Result<_, _>>())
.map_err(|err| sp_blockchain::Error::ChangesTrieAccessFailed(err))
let config_range = ChangesTrieConfigurationRange {
config: &config,
zero: config_zero.clone(),
end: config_end.map(|(config_end_number, _)| config_end_number),
};
let result_range: Vec<(NumberFor<Block>, u32)> = key_changes::<HasherFor<Block>, _>(
config_range,
storage.storage(),
range_first,
&range_anchor,
best_number,
storage_key.as_ref().map(|x| &x.0[..]),
&key.0)
.and_then(|r| r.map(|r| r.map(|(block, tx)| (block, tx))).collect::<Result<_, _>>())
.map_err(|err| sp_blockchain::Error::ChangesTrieAccessFailed(err))?;
result.extend(result_range);
}
Ok(result)
}
/// Get proof for computation of (block, extrinsic) pairs where key has been changed at given blocks range.
@@ -550,11 +571,13 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
}
}
let (config, storage) = self.require_changes_trie()?;
let first_number = self.backend.blockchain()
.expect_block_number_from_id(&BlockId::Hash(first))?;
let (storage, configs) = self.require_changes_trie(first_number, last, true)?;
let min_number = self.backend.blockchain().expect_block_number_from_id(&BlockId::Hash(min))?;
let recording_storage = AccessedRootsRecorder::<Block> {
storage,
storage: storage.storage(),
min: min_number,
required_roots_proofs: Mutex::new(BTreeMap::new()),
};
@@ -564,31 +587,31 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
self.backend.blockchain().expect_block_number_from_id(&BlockId::Hash(max))?,
);
// FIXME: remove this in https://github.com/paritytech/substrate/pull/3201
let config_range = ChangesTrieConfigurationRange {
config: &config,
zero: Zero::zero(),
end: None,
};
// fetch key changes proof
let first_number = self.backend.blockchain()
.expect_block_number_from_id(&BlockId::Hash(first))?;
let last_number = self.backend.blockchain()
.expect_block_number_from_id(&BlockId::Hash(last))?;
let key_changes_proof = key_changes_proof::<HasherFor<Block>, _>(
config_range,
&recording_storage,
first_number,
&ChangesTrieAnchorBlockId {
hash: convert_hash(&last),
number: last_number,
},
max_number,
storage_key.as_ref().map(|sk| sk.0.as_slice()),
&key.0,
)
.map_err(|err| sp_blockchain::Error::from(sp_blockchain::Error::ChangesTrieAccessFailed(err)))?;
let mut proof = Vec::new();
for (config_zero, config_end, config) in configs {
let last_number = self.backend.blockchain()
.expect_block_number_from_id(&BlockId::Hash(last))?;
let config_range = ChangesTrieConfigurationRange {
config: &config,
zero: config_zero,
end: config_end.map(|(config_end_number, _)| config_end_number),
};
let proof_range = key_changes_proof::<HasherFor<Block>, _>(
config_range,
&recording_storage,
first_number,
&ChangesTrieAnchorBlockId {
hash: convert_hash(&last),
number: last_number,
},
max_number,
storage_key.as_ref().map(|x| &x.0[..]),
&key.0,
)
.map_err(|err| sp_blockchain::Error::ChangesTrieAccessFailed(err))?;
proof.extend(proof_range);
}
// now gather proofs for all changes tries roots that were touched during key_changes_proof
// execution AND are unknown (i.e. replaced with CHT) to the requester
@@ -597,7 +620,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
Ok(ChangesProof {
max_block: max_number,
proof: key_changes_proof,
proof,
roots: roots.into_iter().map(|(n, h)| (n, convert_hash(&h))).collect(),
roots_proof,
})
@@ -650,14 +673,45 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
Ok(proof)
}
/// Returns changes trie configuration and storage or an error if it is not supported.
fn require_changes_trie(&self) -> sp_blockchain::Result<(ChangesTrieConfiguration, &B::ChangesTrieStorage)> {
let config = self.changes_trie_config()?;
let storage = self.backend.changes_trie_storage();
match (config, storage) {
(Some(config), Some(storage)) => Ok((config, storage)),
_ => Err(sp_blockchain::Error::ChangesTriesNotSupported.into()),
/// Returns changes trie storage and all configurations that have been active in the range [first; last].
///
/// Configurations are returned in descending order (and obviously never overlap).
/// If fail_if_disabled is false, returns maximal consequent configurations ranges, starting from last and
/// stopping on either first, or when CT have been disabled.
/// If fail_if_disabled is true, fails when there's a subrange where CT have been disabled
/// inside first..last blocks range.
fn require_changes_trie(
&self,
first: NumberFor<Block>,
last: Block::Hash,
fail_if_disabled: bool,
) -> sp_blockchain::Result<(
&dyn PrunableStateChangesTrieStorage<Block>,
Vec<(NumberFor<Block>, Option<(NumberFor<Block>, Block::Hash)>, ChangesTrieConfiguration)>,
)> {
let storage = match self.backend.changes_trie_storage() {
Some(storage) => storage,
None => return Err(sp_blockchain::Error::ChangesTriesNotSupported),
};
let mut configs = Vec::with_capacity(1);
let mut current = last;
loop {
let config_range = storage.configuration_at(&BlockId::Hash(current))?;
match config_range.config {
Some(config) => configs.push((config_range.zero.0, config_range.end, config)),
None if !fail_if_disabled => return Ok((storage, configs)),
None => return Err(sp_blockchain::Error::ChangesTriesNotSupported),
}
if config_range.zero.0 < first {
break;
}
current = *self.backend.blockchain().expect_header(BlockId::Hash(config_range.zero.1))?.parent_hash();
}
Ok((storage, configs))
}
/// Create a new block, built on the head of the chain.
@@ -988,10 +1042,14 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
)?;
let state = self.backend.state_at(at)?;
let changes_trie_state = changes_tries_state_at_block(
&at,
self.backend.changes_trie_storage(),
)?;
let gen_storage_changes = runtime_api.into_storage_changes(
&state,
self.backend.changes_trie_storage(),
changes_trie_state.as_ref(),
*parent_hash,
);
@@ -1249,13 +1307,6 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
Ok(uncles)
}
fn changes_trie_config(&self) -> Result<Option<ChangesTrieConfiguration>, Error> {
Ok(self.backend.state_at(BlockId::Number(self.backend.blockchain().info().best_number))?
.storage(well_known_keys::CHANGES_TRIE_CONFIG)
.map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))?
.and_then(|c| Decode::decode(&mut &*c).ok()))
}
/// Prepare in-memory header that is used in execution environment.
fn prepare_environment_block(&self, parent: &BlockId<Block>) -> sp_blockchain::Result<Block::Header> {
let parent_header = self.backend.blockchain().expect_header(*parent)?;
@@ -1871,7 +1922,8 @@ pub(crate) mod tests {
// prepare client ang import blocks
let mut local_roots = Vec::new();
let mut remote_client = TestClientBuilder::new().set_support_changes_trie(true).build();
let config = Some(ChangesTrieConfiguration::new(4, 2));
let mut remote_client = TestClientBuilder::new().changes_trie_config(config).build();
let mut nonces: HashMap<_, u64> = Default::default();
for (i, block_transfers) in blocks_transfers.into_iter().enumerate() {
let mut builder = remote_client.new_block(Default::default()).unwrap();
@@ -2907,13 +2959,15 @@ pub(crate) mod tests {
.unwrap().build().unwrap().block;
client.import(BlockOrigin::Own, b2.clone()).unwrap();
// prepare B3 before we finalize A2, because otherwise we won't be able to
// read changes trie configuration after A2 is finalized
let b3 = client.new_block_at(&BlockId::Hash(b2.hash()), Default::default(), false)
.unwrap().build().unwrap().block;
// we will finalize A2 which should make it impossible to import a new
// B3 at the same height but that doesnt't include it
ClientExt::finalize_block(&client, BlockId::Hash(a2.hash()), None).unwrap();
let b3 = client.new_block_at(&BlockId::Hash(b2.hash()), Default::default(), false)
.unwrap().build().unwrap().block;
let import_err = client.import(BlockOrigin::Own, b3).err().unwrap();
let expected_err = ConsensusError::ClientImport(
sp_blockchain::Error::NotInFinalizedChain.to_string()
@@ -3050,4 +3104,104 @@ pub(crate) mod tests {
check_block_b1.parent_hash = H256::random();
assert_eq!(client.check_block(check_block_b1.clone()).unwrap(), ImportResult::UnknownParent);
}
#[test]
fn imports_blocks_with_changes_tries_config_change() {
// create client with initial 4^2 configuration
let mut client = TestClientBuilder::with_default_backend()
.changes_trie_config(Some(ChangesTrieConfiguration {
digest_interval: 4,
digest_levels: 2,
})).build();
// ===================================================================
// blocks 1,2,3,4,5,6,7,8,9,10 are empty
// block 11 changes the key
// block 12 is the L1 digest that covers this change
// blocks 13,14,15,16,17,18,19,20,21,22 are empty
// block 23 changes the configuration to 5^1 AND is skewed digest
// ===================================================================
// blocks 24,25 are changing the key
// block 26 is empty
// block 27 changes the key
// block 28 is the L1 digest (NOT SKEWED!!!) that covers changes AND changes configuration to 3^1
// ===================================================================
// block 29 is empty
// block 30 changes the key
// block 31 is L1 digest that covers this change
// ===================================================================
(1..11).for_each(|number| {
let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false)
.unwrap().build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(11..12).for_each(|number| {
let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap();
block.push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())).unwrap();
let block = block.build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(12..23).for_each(|number| {
let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false)
.unwrap().build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(23..24).for_each(|number| {
let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap();
block.push_changes_trie_configuration_update(Some(ChangesTrieConfiguration {
digest_interval: 5,
digest_levels: 1,
})).unwrap();
let block = block.build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(24..26).for_each(|number| {
let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap();
block.push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())).unwrap();
let block = block.build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(26..27).for_each(|number| {
let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false)
.unwrap().build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(27..28).for_each(|number| {
let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap();
block.push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())).unwrap();
let block = block.build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(28..29).for_each(|number| {
let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap();
block.push_changes_trie_configuration_update(Some(ChangesTrieConfiguration {
digest_interval: 3,
digest_levels: 1,
})).unwrap();
let block = block.build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(29..30).for_each(|number| {
let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false)
.unwrap().build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(30..31).for_each(|number| {
let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap();
block.push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())).unwrap();
let block = block.build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
(31..32).for_each(|number| {
let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false)
.unwrap().build().unwrap().block;
client.import(BlockOrigin::Own, block).unwrap();
});
// now check that configuration cache works
assert_eq!(
client.key_changes(1, BlockId::Number(31), None, &StorageKey(vec![42])).unwrap(),
vec![(30, 0), (27, 0), (25, 0), (24, 0), (11, 0)]
);
}
}
substrate/client/src/genesis.rs
+10 -10
View File
@@ -45,7 +45,7 @@ mod tests {
use codec::{Encode, Decode, Joiner};
use sc_executor::native_executor_instance;
use sp_state_machine::{
StateMachine, OverlayedChanges, ExecutionStrategy, InMemoryChangesTrieStorage,
StateMachine, OverlayedChanges, ExecutionStrategy,
InMemoryBackend,
};
use substrate_test_runtime_client::{
@@ -92,7 +92,7 @@ mod tests {
StateMachine::new(
backend,
Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
sp_state_machine::disabled_changes_trie_state::<_, u64>(),
&mut overlay,
&executor(),
"Core_initialize_block",
@@ -105,7 +105,7 @@ mod tests {
for tx in transactions.iter() {
StateMachine::new(
backend,
Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
sp_state_machine::disabled_changes_trie_state::<_, u64>(),
&mut overlay,
&executor(),
"BlockBuilder_apply_extrinsic",
@@ -118,7 +118,7 @@ mod tests {
let ret_data = StateMachine::new(
backend,
Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
sp_state_machine::disabled_changes_trie_state::<_, u64>(),
&mut overlay,
&executor(),
"BlockBuilder_finalize_block",
@@ -150,7 +150,7 @@ mod tests {
#[test]
fn construct_genesis_should_work_with_native() {
let mut storage = GenesisConfig::new(
false,
None,
vec![Sr25519Keyring::One.public().into(), Sr25519Keyring::Two.public().into()],
vec![AccountKeyring::One.into(), AccountKeyring::Two.into()],
1000,
@@ -165,7 +165,7 @@ mod tests {
let mut overlay = OverlayedChanges::default();
let _ = StateMachine::new(
&backend,
Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
sp_state_machine::disabled_changes_trie_state::<_, u64>(),
&mut overlay,
&executor(),
"Core_execute_block",
@@ -178,7 +178,7 @@ mod tests {
#[test]
fn construct_genesis_should_work_with_wasm() {
let mut storage = GenesisConfig::new(false,
let mut storage = GenesisConfig::new(None,
vec![Sr25519Keyring::One.public().into(), Sr25519Keyring::Two.public().into()],
vec![AccountKeyring::One.into(), AccountKeyring::Two.into()],
1000,
@@ -193,7 +193,7 @@ mod tests {
let mut overlay = OverlayedChanges::default();
let _ = StateMachine::new(
&backend,
Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
sp_state_machine::disabled_changes_trie_state::<_, u64>(),
&mut overlay,
&executor(),
"Core_execute_block",
@@ -206,7 +206,7 @@ mod tests {
#[test]
fn construct_genesis_with_bad_transaction_should_panic() {
let mut storage = GenesisConfig::new(false,
let mut storage = GenesisConfig::new(None,
vec![Sr25519Keyring::One.public().into(), Sr25519Keyring::Two.public().into()],
vec![AccountKeyring::One.into(), AccountKeyring::Two.into()],
68,
@@ -221,7 +221,7 @@ mod tests {
let mut overlay = OverlayedChanges::default();
let r = StateMachine::new(
&backend,
Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
sp_state_machine::disabled_changes_trie_state::<_, u64>(),
&mut overlay,
&executor(),
"Core_execute_block",
substrate/client/src/in_mem.rs
+7 -89
View File
@@ -16,22 +16,17 @@
//! In memory client backend
use std::collections::{HashMap, HashSet};
use std::collections::HashMap;
use std::sync::Arc;
use parking_lot::RwLock;
use sp_core::{ChangesTrieConfiguration, storage::well_known_keys};
use sp_core::storage::well_known_keys;
use sp_core::offchain::storage::{
InMemOffchainStorage as OffchainStorage
};
use sp_runtime::generic::{BlockId, DigestItem};
use sp_runtime::generic::BlockId;
use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, NumberFor, HasherFor};
use sp_runtime::{Justification, Storage};
use sp_state_machine::{
InMemoryChangesTrieStorage, ChangesTrieAnchorBlockId, ChangesTrieTransaction,
InMemoryBackend, Backend as StateBackend,
};
use hash_db::Prefix;
use sp_trie::MemoryDB;
use sp_state_machine::{ChangesTrieTransaction, InMemoryBackend, Backend as StateBackend};
use sp_blockchain::{CachedHeaderMetadata, HeaderMetadata};
use sc_client_api::{
@@ -466,7 +461,6 @@ pub struct BlockImportOperation<Block: BlockT> {
pending_cache: HashMap<CacheKeyId, Vec<u8>>,
old_state: InMemoryBackend<HasherFor<Block>>,
new_state: Option<InMemoryBackend<HasherFor<Block>>>,
changes_trie_update: Option<MemoryDB<HasherFor<Block>>>,
aux: Vec<(Vec<u8>, Option<Vec<u8>>)>,
finalized_blocks: Vec<(BlockId<Block>, Option<Justification>)>,
set_head: Option<BlockId<Block>>,
@@ -510,9 +504,8 @@ impl<Block: BlockT> backend::BlockImportOperation<Block> for BlockImportOperatio
fn update_changes_trie(
&mut self,
update: ChangesTrieTransaction<HasherFor<Block>, NumberFor<Block>>,
_update: ChangesTrieTransaction<HasherFor<Block>, NumberFor<Block>>,
) -> sp_blockchain::Result<()> {
self.changes_trie_update = Some(update.0);
Ok(())
}
@@ -569,7 +562,6 @@ impl<Block: BlockT> backend::BlockImportOperation<Block> for BlockImportOperatio
/// > struct for testing purposes. Do **NOT** use in production.
pub struct Backend<Block: BlockT> where Block::Hash: Ord {
states: RwLock<HashMap<Block::Hash, InMemoryBackend<HasherFor<Block>>>>,
changes_trie_storage: ChangesTrieStorage<Block>,
blockchain: Blockchain<Block>,
import_lock: RwLock<()>,
}
@@ -579,7 +571,6 @@ impl<Block: BlockT> Backend<Block> where Block::Hash: Ord {
pub fn new() -> Self {
Backend {
states: RwLock::new(HashMap::new()),
changes_trie_storage: ChangesTrieStorage(InMemoryChangesTrieStorage::new()),
blockchain: Blockchain::new(),
import_lock: Default::default(),
}
@@ -606,7 +597,6 @@ impl<Block: BlockT> backend::Backend<Block> for Backend<Block> where Block::Hash
type BlockImportOperation = BlockImportOperation<Block>;
type Blockchain = Blockchain<Block>;
type State = InMemoryBackend<HasherFor<Block>>;
type ChangesTrieStorage = ChangesTrieStorage<Block>;
type OffchainStorage = OffchainStorage;
fn begin_operation(&self) -> sp_blockchain::Result<Self::BlockImportOperation> {
@@ -616,7 +606,6 @@ impl<Block: BlockT> backend::Backend<Block> for Backend<Block> where Block::Hash
pending_cache: Default::default(),
old_state,
new_state: None,
changes_trie_update: None,
aux: Default::default(),
finalized_blocks: Default::default(),
set_head: None,
@@ -647,17 +636,6 @@ impl<Block: BlockT> backend::Backend<Block> for Backend<Block> where Block::Hash
self.states.write().insert(hash, operation.new_state.unwrap_or_else(|| old_state.clone()));
let maybe_changes_trie_root = header.digest().log(DigestItem::as_changes_trie_root).cloned();
if let Some(changes_trie_root) = maybe_changes_trie_root {
if let Some(changes_trie_update) = operation.changes_trie_update {
self.changes_trie_storage.0.insert(
*header.number(),
changes_trie_root,
changes_trie_update
);
}
}
self.blockchain.insert(hash, header, justification, body, pending_block.state)?;
}
@@ -688,8 +666,8 @@ impl<Block: BlockT> backend::Backend<Block> for Backend<Block> where Block::Hash
None
}
fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage> {
Some(&self.changes_trie_storage)
fn changes_trie_storage(&self) -> Option<&dyn backend::PrunableStateChangesTrieStorage<Block>> {
None
}
fn offchain_storage(&self) -> Option<Self::OffchainStorage> {
@@ -737,66 +715,6 @@ impl<Block: BlockT> backend::RemoteBackend<Block> for Backend<Block> where Block
}
}
/// Prunable in-memory changes trie storage.
pub struct ChangesTrieStorage<Block: BlockT>(
InMemoryChangesTrieStorage<HasherFor<Block>, NumberFor<Block>>
);
impl<Block: BlockT> backend::PrunableStateChangesTrieStorage<Block> for ChangesTrieStorage<Block> {
fn oldest_changes_trie_block(
&self,
_config: &ChangesTrieConfiguration,
_best_finalized: NumberFor<Block>,
) -> NumberFor<Block> {
Zero::zero()
}
}
impl<Block: BlockT> sp_state_machine::ChangesTrieRootsStorage<HasherFor<Block>, NumberFor<Block>> for
ChangesTrieStorage<Block>
{
fn build_anchor(
&self,
_hash: Block::Hash,
) -> Result<sp_state_machine::ChangesTrieAnchorBlockId<Block::Hash, NumberFor<Block>>, String> {
Err("Dummy implementation".into())
}
fn root(
&self,
_anchor: &ChangesTrieAnchorBlockId<Block::Hash, NumberFor<Block>>,
_block: NumberFor<Block>,
) -> Result<Option<Block::Hash>, String> {
Err("Dummy implementation".into())
}
}
impl<Block: BlockT> sp_state_machine::ChangesTrieStorage<HasherFor<Block>, NumberFor<Block>> for
ChangesTrieStorage<Block>
{
fn as_roots_storage(&self)
-> &dyn sp_state_machine::ChangesTrieRootsStorage<HasherFor<Block>, NumberFor<Block>>
{
self
}
fn with_cached_changed_keys(
&self,
_root: &Block::Hash,
_functor: &mut dyn FnMut(&HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>),
) -> bool {
false
}
fn get(
&self,
key: &Block::Hash,
prefix: Prefix,
) -> Result<Option<sp_state_machine::DBValue>, String> {
self.0.get(key, prefix)
}
}
/// Check that genesis storage is valid.
pub fn check_genesis_storage(storage: &Storage) -> sp_blockchain::Result<()> {
if storage.top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) {
substrate/client/src/light/backend.rs
+19 -5
View File
@@ -21,19 +21,22 @@ use std::collections::HashMap;
use std::sync::Arc;
use parking_lot::RwLock;
use sp_core::storage::{ChildInfo, OwnedChildInfo};
use codec::{Decode, Encode};
use sp_core::ChangesTrieConfiguration;
use sp_core::storage::{well_known_keys, ChildInfo, OwnedChildInfo};
use sp_core::offchain::storage::InMemOffchainStorage;
use sp_state_machine::{
Backend as StateBackend, TrieBackend, InMemoryBackend, ChangesTrieTransaction
};
use sp_runtime::{generic::BlockId, Justification, Storage};
use sp_runtime::traits::{Block as BlockT, NumberFor, Zero, Header, HasherFor};
use crate::in_mem::{self, check_genesis_storage};
use crate::in_mem::check_genesis_storage;
use sp_blockchain::{Error as ClientError, Result as ClientResult};
use sc_client_api::{
backend::{
AuxStore, Backend as ClientBackend, BlockImportOperation, RemoteBackend, NewBlockState,
StorageCollection, ChildStorageCollection,
StorageCollection, ChildStorageCollection, PrunableStateChangesTrieStorage,
},
blockchain::{
HeaderBackend as BlockchainHeaderBackend, well_known_cache_keys,
@@ -62,6 +65,7 @@ pub struct ImportOperation<Block: BlockT, S> {
finalized_blocks: Vec<BlockId<Block>>,
set_head: Option<BlockId<Block>>,
storage_update: Option<InMemoryBackend<HasherFor<Block>>>,
changes_trie_config_update: Option<Option<ChangesTrieConfiguration>>,
_phantom: std::marker::PhantomData<S>,
}
@@ -115,7 +119,6 @@ impl<S, Block> ClientBackend<Block> for Backend<S, HasherFor<Block>>
type BlockImportOperation = ImportOperation<Block, S>;
type Blockchain = Blockchain<S>;
type State = GenesisOrUnavailableState<HasherFor<Block>>;
type ChangesTrieStorage = in_mem::ChangesTrieStorage<Block>;
type OffchainStorage = InMemOffchainStorage;
fn begin_operation(&self) -> ClientResult<Self::BlockImportOperation> {
@@ -127,6 +130,7 @@ impl<S, Block> ClientBackend<Block> for Backend<S, HasherFor<Block>>
finalized_blocks: Vec::new(),
set_head: None,
storage_update: None,
changes_trie_config_update: None,
_phantom: Default::default(),
})
}
@@ -148,6 +152,9 @@ impl<S, Block> ClientBackend<Block> for Backend<S, HasherFor<Block>>
if let Some(header) = operation.header {
let is_genesis_import = header.number().is_zero();
if let Some(new_config) = operation.changes_trie_config_update {
operation.cache.insert(well_known_cache_keys::CHANGES_TRIE_CONFIG, new_config.encode());
}
self.blockchain.storage().import_header(
header,
operation.cache,
@@ -194,7 +201,7 @@ impl<S, Block> ClientBackend<Block> for Backend<S, HasherFor<Block>>
self.blockchain.storage().usage_info()
}
fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage> {
fn changes_trie_storage(&self) -> Option<&dyn PrunableStateChangesTrieStorage<Block>> {
None
}
@@ -296,6 +303,13 @@ impl<S, Block> BlockImportOperation<Block> for ImportOperation<Block, S>
fn reset_storage(&mut self, input: Storage) -> ClientResult<Block::Hash> {
check_genesis_storage(&input)?;
// changes trie configuration
let changes_trie_config = input.top.iter()
.find(|(k, _)| &k[..] == well_known_keys::CHANGES_TRIE_CONFIG)
.map(|(_, v)| Decode::decode(&mut &v[..])
.expect("changes trie configuration is encoded properly at genesis"));
self.changes_trie_config_update = Some(changes_trie_config);
// this is only called when genesis block is imported => shouldn't be performance bottleneck
let mut storage: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, _> = HashMap::new();
storage.insert(None, input.top);
substrate/client/src/light/call_executor.rs
+2 -2
View File
@@ -259,7 +259,7 @@ fn check_execution_proof_with_make_header<Header, E, H, MakeNextHeader: Fn(&Head
let mut changes = OverlayedChanges::default();
let trie_backend = create_proof_check_backend(root, remote_proof)?;
let next_header = make_next_header(&request.header);
execution_proof_check_on_trie_backend::<H, _>(
execution_proof_check_on_trie_backend::<H, Header::Number, _>(
&trie_backend,
&mut changes,
executor,
@@ -268,7 +268,7 @@ fn check_execution_proof_with_make_header<Header, E, H, MakeNextHeader: Fn(&Head
)?;
// execute method
execution_proof_check_on_trie_backend::<H, _>(
execution_proof_check_on_trie_backend::<H, Header::Number, _>(
&trie_backend,
&mut changes,
executor,
substrate/client/src/light/fetcher.rs
+50 -31
View File
@@ -25,12 +25,12 @@ use codec::{Decode, Encode};
use sp_core::{convert_hash, traits::CodeExecutor};
use sp_runtime::traits::{
Block as BlockT, Header as HeaderT, Hash, HashFor, NumberFor,
SimpleArithmetic, CheckedConversion, Zero,
SimpleArithmetic, CheckedConversion,
};
use sp_state_machine::{
ChangesTrieRootsStorage, ChangesTrieAnchorBlockId, ChangesTrieConfigurationRange,
TrieBackend, read_proof_check, key_changes_proof_check, create_proof_check_backend_storage,
read_child_proof_check,
InMemoryChangesTrieStorage, TrieBackend, read_proof_check, key_changes_proof_check_with_db,
create_proof_check_backend_storage, read_child_proof_check,
};
pub use sp_state_machine::StorageProof;
use sp_blockchain::{Error as ClientError, Result as ClientResult};
@@ -113,30 +113,34 @@ impl<E, H, B: BlockT, S: BlockchainStorage<B>> LightDataChecker<E, H, B, S> {
)?;
}
// FIXME: remove this in https://github.com/paritytech/substrate/pull/3201
let changes_trie_config_range = ChangesTrieConfigurationRange {
config: &request.changes_trie_config,
zero: Zero::zero(),
end: None,
};
// and now check the key changes proof + get the changes
key_changes_proof_check::<H, _>(
changes_trie_config_range,
&RootsStorage {
roots: (request.tries_roots.0, &request.tries_roots.2),
prev_roots: remote_roots,
},
remote_proof,
request.first_block.0,
&ChangesTrieAnchorBlockId {
hash: convert_hash(&request.last_block.1),
number: request.last_block.0,
},
remote_max_block,
request.storage_key.as_ref().map(Vec::as_slice),
&request.key)
.map_err(|err| ClientError::ChangesTrieAccessFailed(err))
let mut result = Vec::new();
let proof_storage = InMemoryChangesTrieStorage::with_proof(remote_proof);
for config_range in &request.changes_trie_configs {
let result_range = key_changes_proof_check_with_db::<H, _>(
ChangesTrieConfigurationRange {
config: config_range.config.as_ref().ok_or(ClientError::ChangesTriesNotSupported)?,
zero: config_range.zero.0,
end: config_range.end.map(|(n, _)| n),
},
&RootsStorage {
roots: (request.tries_roots.0, &request.tries_roots.2),
prev_roots: &remote_roots,
},
&proof_storage,
request.first_block.0,
&ChangesTrieAnchorBlockId {
hash: convert_hash(&request.last_block.1),
number: request.last_block.0,
},
remote_max_block,
request.storage_key.as_ref().map(Vec::as_slice),
&request.key)
.map_err(|err| ClientError::ChangesTrieAccessFailed(err))?;
result.extend(result_range);
}
Ok(result)
}
/// Check CHT-based proof for changes tries roots.
@@ -284,7 +288,7 @@ impl<E, Block, H, S> FetchChecker<Block> for LightDataChecker<E, H, Block, S>
/// A view of BTreeMap<Number, Hash> as a changes trie roots storage.
struct RootsStorage<'a, Number: SimpleArithmetic, Hash: 'a> {
roots: (Number, &'a [Hash]),
prev_roots: BTreeMap<Number, Hash>,
prev_roots: &'a BTreeMap<Number, Hash>,
}
impl<'a, H, Number, Hash> ChangesTrieRootsStorage<H, Number> for RootsStorage<'a, Number, Hash>
@@ -340,7 +344,7 @@ pub mod tests {
use crate::in_mem::{Blockchain as InMemoryBlockchain};
use crate::light::fetcher::{FetchChecker, LightDataChecker, RemoteHeaderRequest};
use crate::light::blockchain::tests::{DummyStorage, DummyBlockchain};
use sp_core::{blake2_256, Blake2Hasher, H256};
use sp_core::{blake2_256, Blake2Hasher, ChangesTrieConfiguration, H256};
use sp_core::storage::{well_known_keys, StorageKey, ChildInfo};
use sp_runtime::generic::BlockId;
use sp_state_machine::Backend;
@@ -569,8 +573,13 @@ pub mod tests {
// check proof on local client
let local_roots_range = local_roots.clone()[(begin - 1) as usize..].to_vec();
let config = ChangesTrieConfiguration::new(4, 2);
let request = RemoteChangesRequest::<Header> {
changes_trie_config: runtime::changes_trie_config(),
changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange {
zero: (0, Default::default()),
end: None,
config: Some(config),
}],
first_block: (begin, begin_hash),
last_block: (end, end_hash),
max_block: (max, max_hash),
@@ -624,8 +633,13 @@ pub mod tests {
);
// check proof on local client
let config = ChangesTrieConfiguration::new(4, 2);
let request = RemoteChangesRequest::<Header> {
changes_trie_config: runtime::changes_trie_config(),
changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange {
zero: (0, Default::default()),
end: None,
config: Some(config),
}],
first_block: (1, b1),
last_block: (4, b4),
max_block: (4, b4),
@@ -665,8 +679,13 @@ pub mod tests {
begin_hash, end_hash, begin_hash, max_hash, None, &key).unwrap();
let local_roots_range = local_roots.clone()[(begin - 1) as usize..].to_vec();
let config = ChangesTrieConfiguration::new(4, 2);
let request = RemoteChangesRequest::<Header> {
changes_trie_config: runtime::changes_trie_config(),
changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange {
zero: (0, Default::default()),
end: None,
config: Some(config),
}],
first_block: (begin, begin_hash),
last_block: (end, end_hash),
max_block: (max, max_hash),
substrate/frame/system/src/lib.rs
+19 -4
View File
@@ -110,7 +110,7 @@ use sp_runtime::{
},
};
use sp_core::storage::well_known_keys;
use sp_core::{ChangesTrieConfiguration, storage::well_known_keys};
use frame_support::{
decl_module, decl_event, decl_storage, decl_error, storage, Parameter,
traits::{Contains, Get, ModuleToIndex, OnReapAccount},
@@ -121,9 +121,6 @@ use codec::{Encode, Decode};
#[cfg(any(feature = "std", test))]
use sp_io::TestExternalities;
#[cfg(any(feature = "std", test))]
use sp_core::ChangesTrieConfiguration;
pub mod offchain;
/// Handler for when a new account has been created.
@@ -293,6 +290,24 @@ decl_module! {
storage::unhashed::put_raw(well_known_keys::CODE, &code);
}
/// Set the new changes trie configuration.
#[weight = SimpleDispatchInfo::FixedOperational(20_000)]
pub fn set_changes_trie_config(origin, changes_trie_config: Option<ChangesTrieConfiguration>) {
ensure_root(origin)?;
match changes_trie_config.clone() {
Some(changes_trie_config) => storage::unhashed::put_raw(
well_known_keys::CHANGES_TRIE_CONFIG,
&changes_trie_config.encode(),
),
None => storage::unhashed::kill(well_known_keys::CHANGES_TRIE_CONFIG),
}
let log = generic::DigestItem::ChangesTrieSignal(
generic::ChangesTrieSignal::NewConfiguration(changes_trie_config),
);
Self::deposit_log(log.into());
}
/// Set some items of storage.
#[weight = SimpleDispatchInfo::FixedOperational(10_000)]
fn set_storage(origin, items: Vec<KeyValue>) {
substrate/primitives/api/proc-macro/src/impl_runtime_apis.rs
+6 -5
View File
@@ -323,12 +323,13 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
})
}
fn into_storage_changes<
T: #crate_::ChangesTrieStorage<#crate_::HasherFor<Block>, #crate_::NumberFor<Block>>
>(
fn into_storage_changes(
&self,
backend: &Self::StateBackend,
changes_trie_storage: Option<&T>,
changes_trie_state: Option<&#crate_::ChangesTrieState<
#crate_::HasherFor<Block>,
#crate_::NumberFor<Block>,
>>,
parent_hash: Block::Hash,
) -> std::result::Result<
#crate_::StorageChanges<Self::StateBackend, Block>,
@@ -337,7 +338,7 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
self.initialized_block.borrow_mut().take();
self.changes.replace(Default::default()).into_storage_changes(
backend,
changes_trie_storage,
changes_trie_state,
parent_hash,
self.storage_transaction_cache.replace(Default::default()),
)
substrate/primitives/api/src/lib.rs
+3 -3
View File
@@ -36,7 +36,7 @@ extern crate self as sp_api;
#[doc(hidden)]
#[cfg(feature = "std")]
pub use sp_state_machine::{
OverlayedChanges, StorageProof, Backend as StateBackend, ChangesTrieStorage,
OverlayedChanges, StorageProof, Backend as StateBackend, ChangesTrieState,
};
#[doc(hidden)]
#[cfg(feature = "std")]
@@ -325,10 +325,10 @@ pub trait ApiExt<Block: BlockT>: ApiErrorExt {
/// api functions.
///
/// After executing this function, all collected changes are reset.
fn into_storage_changes<T: ChangesTrieStorage<HasherFor<Block>, NumberFor<Block>>>(
fn into_storage_changes(
&self,
backend: &Self::StateBackend,
changes_trie_storage: Option<&T>,
changes_trie_state: Option<&ChangesTrieState<HasherFor<Block>, NumberFor<Block>>>,
parent_hash: Block::Hash,
) -> Result<StorageChanges<Self::StateBackend, Block>, String> where Self: Sized;
}
substrate/primitives/api/test/tests/runtime_calls.rs
+1 -1
View File
@@ -185,7 +185,7 @@ fn record_proof_works() {
// Use the proof backend to execute `execute_block`.
let mut overlay = Default::default();
let executor = NativeExecutor::<LocalExecutor>::new(WasmExecutionMethod::Interpreted, None);
execution_proof_check_on_trie_backend(
execution_proof_check_on_trie_backend::<_, u64, _>(
&backend,
&mut overlay,
&executor,
substrate/primitives/blockchain/src/backend.rs
+3 -1
View File
@@ -228,11 +228,13 @@ pub trait Cache<Block: BlockT>: Send + Sync {
/// Returns cached value by the given key.
///
/// Returned tuple is the range where value has been active and the value itself.
/// Fails if read from cache storage fails or if the value for block is discarded
/// (i.e. if block is earlier that best finalized, but it is not in canonical chain).
fn get_at(
&self,
key: &well_known_cache_keys::Id,
block: &BlockId<Block>,
) -> Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)>;
) -> Result<Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)>>;
}
/// Blockchain info
substrate/primitives/blockchain/src/error.rs
+3
View File
@@ -106,6 +106,9 @@ pub enum Error {
/// Changes tries are not supported.
#[display(fmt = "Changes tries are not supported by the runtime")]
ChangesTriesNotSupported,
/// Error reading changes tries configuration.
#[display(fmt = "Error reading changes tries configuration")]
ErrorReadingChangesTriesConfig,
/// Key changes query has failed.
#[display(fmt = "Failed to check changes proof: {}", _0)]
#[from(ignore)]
substrate/primitives/core/src/changes_trie.rs
+11
View File
@@ -38,6 +38,17 @@ pub struct ChangesTrieConfiguration {
pub digest_levels: u32,
}
/// Substrate changes trie configuration range.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ChangesTrieConfigurationRange<Number, Hash> {
/// Zero block of configuration.
pub zero: (Number, Hash),
/// Last block of configuration (if configuration has been deactivated at some point).
pub end: Option<(Number, Hash)>,
/// The configuration itself. None if changes tries were disabled in this range.
pub config: Option<ChangesTrieConfiguration>,
}
impl ChangesTrieConfiguration {
/// Create new configuration given digest interval and levels.
pub fn new(digest_interval: u32, digest_levels: u32) -> Self {
substrate/primitives/core/src/lib.rs
+1 -1
View File
@@ -75,7 +75,7 @@ mod tests;
pub use self::hash::{H160, H256, H512, convert_hash};
pub use self::uint::U256;
pub use changes_trie::ChangesTrieConfiguration;
pub use changes_trie::{ChangesTrieConfiguration, ChangesTrieConfigurationRange};
#[cfg(feature = "full_crypto")]
pub use crypto::{DeriveJunction, Pair, Public};
substrate/primitives/runtime/src/generic/digest.rs
+59 -3
View File
@@ -23,7 +23,7 @@ use sp_std::prelude::*;
use crate::ConsensusEngineId;
use crate::codec::{Decode, Encode, Input, Error};
use sp_core::RuntimeDebug;
use sp_core::{ChangesTrieConfiguration, RuntimeDebug};
/// Generic header digest.
#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug)]
@@ -97,10 +97,32 @@ pub enum DigestItem<Hash> {
/// by runtimes.
Seal(ConsensusEngineId, Vec<u8>),
/// Digest item that contains signal from changes tries manager to the
/// native code.
ChangesTrieSignal(ChangesTrieSignal),
/// Some other thing. Unsupported and experimental.
Other(Vec<u8>),
}
/// Available changes trie signals.
#[derive(PartialEq, Eq, Clone, Encode, Decode)]
#[cfg_attr(feature = "std", derive(Debug))]
pub enum ChangesTrieSignal {
/// New changes trie configuration is enacted, starting from **next block**.
///
/// The block that emits this signal will contain changes trie (CT) that covers
/// blocks range [BEGIN; current block], where BEGIN is (order matters):
/// - LAST_TOP_LEVEL_DIGEST_BLOCK+1 if top level digest CT has ever been created
/// using current configuration AND the last top level digest CT has been created
/// at block LAST_TOP_LEVEL_DIGEST_BLOCK;
/// - LAST_CONFIGURATION_CHANGE_BLOCK+1 if there has been CT configuration change
/// before and the last configuration change happened at block
/// LAST_CONFIGURATION_CHANGE_BLOCK;
/// - 1 otherwise.
NewConfiguration(Option<ChangesTrieConfiguration>),
}
#[cfg(feature = "std")]
impl<Hash: Encode> serde::Serialize for DigestItem<Hash> {
fn serialize<S>(&self, seq: S) -> Result<S::Ok, S::Error> where S: serde::Serializer {
@@ -141,6 +163,9 @@ pub enum DigestItemRef<'a, Hash: 'a> {
/// Put a Seal on it. This is only used by native code, and is never seen
/// by runtimes.
Seal(&'a ConsensusEngineId, &'a Vec<u8>),
/// Digest item that contains signal from changes tries manager to the
/// native code.
ChangesTrieSignal(&'a ChangesTrieSignal),
/// Any 'non-system' digest item, opaque to the native code.
Other(&'a Vec<u8>),
}
@@ -152,11 +177,12 @@ pub enum DigestItemRef<'a, Hash: 'a> {
#[repr(u32)]
#[derive(Encode, Decode)]
pub enum DigestItemType {
Other = 0,
ChangesTrieRoot = 2,
PreRuntime = 6,
Consensus = 4,
Seal = 5,
Other = 0,
PreRuntime = 6,
ChangesTrieSignal = 7,
}
/// Type of a digest item that contains raw data; this also names the consensus engine ID where
@@ -181,6 +207,7 @@ impl<Hash> DigestItem<Hash> {
DigestItem::PreRuntime(ref v, ref s) => DigestItemRef::PreRuntime(v, s),
DigestItem::Consensus(ref v, ref s) => DigestItemRef::Consensus(v, s),
DigestItem::Seal(ref v, ref s) => DigestItemRef::Seal(v, s),
DigestItem::ChangesTrieSignal(ref s) => DigestItemRef::ChangesTrieSignal(s),
DigestItem::Other(ref v) => DigestItemRef::Other(v),
}
}
@@ -205,6 +232,11 @@ impl<Hash> DigestItem<Hash> {
self.dref().as_seal()
}
/// Returns `Some` if the entry is the `ChangesTrieSignal` entry.
pub fn as_changes_trie_signal(&self) -> Option<&ChangesTrieSignal> {
self.dref().as_changes_trie_signal()
}
/// Returns Some if `self` is a `DigestItem::Other`.
pub fn as_other(&self) -> Option<&[u8]> {
match *self {
@@ -253,6 +285,9 @@ impl<Hash: Decode> Decode for DigestItem<Hash> {
let vals: (ConsensusEngineId, Vec<u8>) = Decode::decode(input)?;
Ok(DigestItem::Seal(vals.0, vals.1))
},
DigestItemType::ChangesTrieSignal => Ok(DigestItem::ChangesTrieSignal(
Decode::decode(input)?,
)),
DigestItemType::Other => Ok(DigestItem::Other(
Decode::decode(input)?,
)),
@@ -293,6 +328,14 @@ impl<'a, Hash> DigestItemRef<'a, Hash> {
}
}
/// Cast this digest item into `ChangesTrieSignal`.
pub fn as_changes_trie_signal(&self) -> Option<&'a ChangesTrieSignal> {
match *self {
DigestItemRef::ChangesTrieSignal(ref changes_trie_signal) => Some(changes_trie_signal),
_ => None,
}
}
/// Cast this digest item into `PreRuntime`
pub fn as_other(&self) -> Option<&'a [u8]> {
match *self {
@@ -342,6 +385,10 @@ impl<'a, Hash: Encode> Encode for DigestItemRef<'a, Hash> {
DigestItemType::PreRuntime.encode_to(&mut v);
(val, data).encode_to(&mut v);
},
DigestItemRef::ChangesTrieSignal(changes_trie_signal) => {
DigestItemType::ChangesTrieSignal.encode_to(&mut v);
changes_trie_signal.encode_to(&mut v);
},
DigestItemRef::Other(val) => {
DigestItemType::Other.encode_to(&mut v);
val.encode_to(&mut v);
@@ -352,6 +399,15 @@ impl<'a, Hash: Encode> Encode for DigestItemRef<'a, Hash> {
}
}
impl ChangesTrieSignal {
/// Try to cast this signal to NewConfiguration.
pub fn as_new_configuration(&self) -> Option<&Option<ChangesTrieConfiguration>> {
match self {
ChangesTrieSignal::NewConfiguration(config) => Some(config),
}
}
}
impl<'a, Hash: Encode> codec::EncodeLike for DigestItemRef<'a, Hash> {}
#[cfg(test)]
substrate/primitives/runtime/src/generic/mod.rs
+1 -1
View File
@@ -33,7 +33,7 @@ pub use self::checked_extrinsic::CheckedExtrinsic;
pub use self::header::Header;
pub use self::block::{Block, SignedBlock, BlockId};
pub use self::digest::{
Digest, DigestItem, DigestItemRef, OpaqueDigestItemId
Digest, DigestItem, DigestItemRef, OpaqueDigestItemId, ChangesTrieSignal,
};
use crate::codec::Encode;
substrate/primitives/state-machine/src/changes_trie/build.rs
+3 -7
View File
@@ -356,7 +356,6 @@ mod test {
OverlayedChanges,
Configuration,
) {
let config = Configuration { digest_interval: 4, digest_levels: 2 };
let backend: InMemoryBackend<_> = vec![
(vec![100], vec![255]),
(vec![101], vec![255]),
@@ -465,8 +464,9 @@ mod test {
].into_iter().collect(), CHILD_INFO_1.to_owned())),
].into_iter().collect(),
},
changes_trie_config: Some(config.clone()),
collect_extrinsics: true,
};
let config = Configuration { digest_interval: 4, digest_levels: 2 };
(backend, storage, changes, config)
}
@@ -553,7 +553,6 @@ mod test {
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![100] }, vec![0, 2]),
]),
]);
}
test_with_zero(0);
@@ -597,7 +596,6 @@ mod test {
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 16, key: vec![100] }, vec![0, 2]),
]),
]);
}
test_with_zero(0);
@@ -706,8 +704,7 @@ mod test {
#[test]
fn cache_is_used_when_changes_trie_is_built() {
let (backend, mut storage, changes, _) = prepare_for_build(0);
let config = changes.changes_trie_config.as_ref().unwrap();
let (backend, mut storage, changes, config) = prepare_for_build(0);
let parent = AnchorBlockId { hash: Default::default(), number: 15 };
// override some actual values from storage with values from the cache
@@ -770,6 +767,5 @@ mod test {
InputPair::DigestIndex(DigestIndex { block: 16u64, key: vec![106] }, vec![4]),
],
);
}
}
substrate/primitives/state-machine/src/changes_trie/mod.rs
+67 -19
View File
@@ -63,7 +63,7 @@ pub use self::changes_iterator::{
key_changes, key_changes_proof,
key_changes_proof_check, key_changes_proof_check_with_db,
};
pub use self::prune::{prune, oldest_non_pruned_trie};
pub use self::prune::prune;
use std::collections::{HashMap, HashSet};
use std::convert::TryInto;
@@ -121,6 +121,18 @@ pub struct AnchorBlockId<Hash: std::fmt::Debug, Number: BlockNumber> {
pub number: Number,
}
/// Changes tries state at some block.
pub struct State<'a, H, Number> {
/// Configuration that is active at given block.
pub config: Configuration,
/// Configuration activation block number. Zero if it is the first coonfiguration on the chain,
/// or number of the block that have emit NewConfiguration signal (thus activating configuration
/// starting from the **next** block).
pub zero: Number,
/// Underlying changes tries storage reference.
pub storage: &'a dyn Storage<H, Number>,
}
/// Changes trie storage. Provides access to trie roots and trie nodes.
pub trait RootsStorage<H: Hasher, Number: BlockNumber>: Send + Sync {
/// Resolve hash of the block into anchor.
@@ -170,14 +182,43 @@ pub struct ConfigurationRange<'a, N> {
pub end: Option<N>,
}
impl<'a, H, Number> State<'a, H, Number> {
/// Create state with given config and storage.
pub fn new(
config: Configuration,
zero: Number,
storage: &'a dyn Storage<H, Number>,
) -> Self {
Self {
config,
zero,
storage,
}
}
}
impl<'a, H, Number: Clone> Clone for State<'a, H, Number> {
fn clone(&self) -> Self {
State {
config: self.config.clone(),
zero: self.zero.clone(),
storage: self.storage,
}
}
}
/// Create state where changes tries are disabled.
pub fn disabled_state<'a, H, Number>() -> Option<State<'a, H, Number>> {
None
}
/// Compute the changes trie root and transaction for given block.
/// Returns Err(()) if unknown `parent_hash` has been passed.
/// Returns Ok(None) if there's no data to perform computation.
/// Panics if background storage returns an error (and `panic_on_storage_error` is `true`) OR
/// if insert to MemoryDB fails.
pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, Number: BlockNumber>(
/// Panics if background storage returns an error OR if insert to MemoryDB fails.
pub fn build_changes_trie<'a, B: Backend<H>, H: Hasher, Number: BlockNumber>(
backend: &B,
storage: Option<&'a S>,
state: Option<&'a State<'a, H, Number>>,
changes: &OverlayedChanges,
parent_hash: H::Out,
panic_on_storage_error: bool,
@@ -185,11 +226,6 @@ pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, N
where
H::Out: Ord + 'static + Encode,
{
let (storage, config) = match (storage, changes.changes_trie_config.as_ref()) {
(Some(storage), Some(config)) => (storage, config),
_ => return Ok(None),
};
/// Panics when `res.is_err() && panic`, otherwise it returns `Err(())` on an error.
fn maybe_panic<R, E: std::fmt::Debug>(
res: std::result::Result<R, E>,
@@ -203,23 +239,35 @@ pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, N
})
}
// FIXME: remove this in https://github.com/paritytech/substrate/pull/3201
let config = ConfigurationRange {
config,
zero: Zero::zero(),
end: None,
// when storage isn't provided, changes tries aren't created
let state = match state {
Some(state) => state,
None => return Ok(None),
};
// build_anchor error should not be considered fatal
let parent = storage.build_anchor(parent_hash).map_err(|_| ())?;
let parent = state.storage.build_anchor(parent_hash).map_err(|_| ())?;
let block = parent.number.clone() + One::one();
// prepare configuration range - we already know zero block. Current block may be the end block if configuration
// has been changed in this block
let is_config_changed = match changes.storage(sp_core::storage::well_known_keys::CHANGES_TRIE_CONFIG) {
Some(Some(new_config)) => new_config != &state.config.encode()[..],
Some(None) => true,
None => false,
};
let config_range = ConfigurationRange {
config: &state.config,
zero: state.zero.clone(),
end: if is_config_changed { Some(block.clone()) } else { None },
};
// storage errors are considered fatal (similar to situations when runtime fetches values from storage)
let (input_pairs, child_input_pairs, digest_input_blocks) = maybe_panic(
prepare_input::<B, H, Number>(
backend,
storage,
config.clone(),
state.storage,
config_range.clone(),
changes,
&parent,
),
@@ -227,7 +275,7 @@ pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, N
)?;
// prepare cached data
let mut cache_action = prepare_cached_build_data(config, block.clone());
let mut cache_action = prepare_cached_build_data(config_range, block.clone());
let needs_changed_keys = cache_action.collects_changed_keys();
cache_action = cache_action.set_digest_input_blocks(digest_input_blocks);
substrate/primitives/state-machine/src/changes_trie/prune.rs
+32 -217
View File
@@ -19,52 +19,26 @@
use hash_db::Hasher;
use sp_trie::Recorder;
use log::warn;
use num_traits::{One, Zero};
use num_traits::One;
use crate::proving_backend::ProvingBackendRecorder;
use crate::trie_backend_essence::TrieBackendEssence;
use crate::changes_trie::{AnchorBlockId, Configuration, Storage, BlockNumber};
use crate::changes_trie::{AnchorBlockId, Storage, BlockNumber};
use crate::changes_trie::storage::TrieBackendAdapter;
use crate::changes_trie::input::{ChildIndex, InputKey};
use codec::{Decode, Codec};
/// Get number of oldest block for which changes trie is not pruned
/// given changes trie configuration, pruning parameter and number of
/// best finalized block.
pub fn oldest_non_pruned_trie<Number: BlockNumber>(
config: &Configuration,
min_blocks_to_keep: Number,
best_finalized_block: Number,
) -> Number {
let max_digest_interval = config.max_digest_interval();
let best_finalized_block_rem = best_finalized_block.clone() % max_digest_interval.into();
let max_digest_block = best_finalized_block - best_finalized_block_rem;
match pruning_range(config, min_blocks_to_keep, max_digest_block) {
Some((_, last_pruned_block)) => last_pruned_block + One::one(),
None => One::one(),
}
}
/// Prune obsolete changes tries. Pruning happens at the same block, where highest
/// level digest is created. Pruning guarantees to save changes tries for last
/// `min_blocks_to_keep` blocks. We only prune changes tries at `max_digest_interval`
/// ranges.
/// Returns MemoryDB that contains all deleted changes tries nodes.
pub fn prune<S: Storage<H, Number>, H: Hasher, Number: BlockNumber, F: FnMut(H::Out)>(
config: &Configuration,
storage: &S,
min_blocks_to_keep: Number,
pub fn prune<H: Hasher, Number: BlockNumber, F: FnMut(H::Out)>(
storage: &dyn Storage<H, Number>,
first: Number,
last: Number,
current_block: &AnchorBlockId<H::Out, Number>,
mut remove_trie_node: F,
) where H::Out: Codec {
// select range for pruning
let (first, last) = match pruning_range(config, min_blocks_to_keep, current_block.number.clone()) {
Some((first, last)) => (first, last),
None => return,
};
// delete changes trie for every block in range
// FIXME: limit `max_digest_interval` so that this cycle won't involve huge ranges
let mut block = first;
loop {
if block >= last.clone() + One::one() {
@@ -112,8 +86,8 @@ pub fn prune<S: Storage<H, Number>, H: Hasher, Number: BlockNumber, F: FnMut(H::
}
// Prune a trie.
fn prune_trie<S: Storage<H, Number>, H: Hasher, Number: BlockNumber, F: FnMut(H::Out)>(
storage: &S,
fn prune_trie<H: Hasher, Number: BlockNumber, F: FnMut(H::Out)>(
storage: &dyn Storage<H, Number>,
root: H::Out,
remove_trie_node: &mut F,
) where H::Out: Codec {
@@ -136,100 +110,26 @@ fn prune_trie<S: Storage<H, Number>, H: Hasher, Number: BlockNumber, F: FnMut(H:
}
}
/// Select blocks range (inclusive from both ends) for pruning changes tries in.
fn pruning_range<Number: BlockNumber>(
config: &Configuration,
min_blocks_to_keep: Number,
block: Number,
) -> Option<(Number, Number)> {
// compute number of changes tries we actually want to keep
let (prune_interval, blocks_to_keep) = if config.is_digest_build_enabled() {
// we only CAN prune at block where max-level-digest is created
let max_digest_interval = match config.digest_level_at_block(Zero::zero(), block.clone()) {
Some((digest_level, digest_interval, _)) if digest_level == config.digest_levels =>
digest_interval,
_ => return None,
};
// compute maximal number of high-level digests to keep
let max_digest_intervals_to_keep = max_digest_intervals_to_keep(min_blocks_to_keep, max_digest_interval);
// number of blocks BEFORE current block where changes tries are not pruned
(
max_digest_interval,
max_digest_intervals_to_keep.checked_mul(&max_digest_interval.into())
)
} else {
(
1,
Some(min_blocks_to_keep)
)
};
// last block for which changes trie is pruned
let last_block_to_prune = blocks_to_keep.and_then(|b| block.checked_sub(&b));
let first_block_to_prune = last_block_to_prune
.clone()
.and_then(|b| b.checked_sub(&prune_interval.into()));
last_block_to_prune
.and_then(|last| first_block_to_prune.map(|first| (first + One::one(), last)))
}
/// Select pruning delay for the changes tries. To make sure we could build a changes
/// trie at block B, we need an access to previous:
/// max_digest_interval = config.digest_interval ^ config.digest_levels
/// blocks. So we can only prune blocks that are earlier than B - max_digest_interval.
/// The pruning_delay stands for number of max_digest_interval-s that we want to keep:
/// 0 or 1: means that only last changes trie is guaranteed to exists;
/// 2: the last changes trie + previous changes trie
/// ...
fn max_digest_intervals_to_keep<Number: BlockNumber>(
min_blocks_to_keep: Number,
max_digest_interval: u32,
) -> Number {
// config.digest_level_at_block ensures that it is not zero
debug_assert!(max_digest_interval != 0);
let max_digest_intervals_to_keep = min_blocks_to_keep / max_digest_interval.into();
if max_digest_intervals_to_keep.is_zero() {
One::one()
} else {
max_digest_intervals_to_keep
}
}
#[cfg(test)]
mod tests {
use std::collections::HashSet;
use sp_trie::MemoryDB;
use sp_core::Blake2Hasher;
use sp_core::{H256, Blake2Hasher};
use crate::backend::insert_into_memory_db;
use crate::changes_trie::storage::InMemoryStorage;
use codec::Encode;
use super::*;
fn config(interval: u32, levels: u32) -> Configuration {
Configuration {
digest_interval: interval,
digest_levels: levels,
}
}
fn prune_by_collect<S: Storage<H, u64>, H: Hasher>(
config: &Configuration,
storage: &S,
min_blocks_to_keep: u64,
fn prune_by_collect(
storage: &dyn Storage<Blake2Hasher, u64>,
first: u64,
last: u64,
current_block: u64,
) -> HashSet<H::Out> where H::Out: Codec {
) -> HashSet<H256> {
let mut pruned_trie_nodes = HashSet::new();
prune(
config,
storage,
min_blocks_to_keep,
&AnchorBlockId { hash: Default::default(), number: current_block },
|node| { pruned_trie_nodes.insert(node); },
);
let anchor = AnchorBlockId { hash: Default::default(), number: current_block };
prune(storage, first, last, &anchor,
|node| { pruned_trie_nodes.insert(node); });
pruned_trie_nodes
}
@@ -243,7 +143,9 @@ mod tests {
&mut mdb1, vec![(vec![10], vec![20])]).unwrap();
let mut mdb2 = MemoryDB::<Blake2Hasher>::default();
let root2 = insert_into_memory_db::<Blake2Hasher, _>(
&mut mdb2, vec![(vec![11], vec![21]), (vec![12], vec![22])]).unwrap();
&mut mdb2,
vec![(vec![11], vec![21]), (vec![12], vec![22])],
).unwrap();
let mut mdb3 = MemoryDB::<Blake2Hasher>::default();
let ch_root3 = insert_into_memory_db::<Blake2Hasher, _>(
&mut mdb3, vec![(vec![110], vec![120])]).unwrap();
@@ -254,7 +156,9 @@ mod tests {
]).unwrap();
let mut mdb4 = MemoryDB::<Blake2Hasher>::default();
let root4 = insert_into_memory_db::<Blake2Hasher, _>(
&mut mdb4, vec![(vec![15], vec![25])]).unwrap();
&mut mdb4,
vec![(vec![15], vec![25])],
).unwrap();
let storage = InMemoryStorage::new();
storage.insert(65, root1, mdb1);
storage.insert(66, root2, mdb2);
@@ -264,109 +168,20 @@ mod tests {
storage
}
// l1-digest is created every 2 blocks
// l2-digest is created every 4 blocks
// we do not want to keep any additional changes tries
// => only one l2-digest is saved AND it is pruned once next is created
let config = Configuration { digest_interval: 2, digest_levels: 2 };
let storage = prepare_storage();
assert!(prune_by_collect(&config, &storage, 0, 69).is_empty());
assert!(prune_by_collect(&config, &storage, 0, 70).is_empty());
assert!(prune_by_collect(&config, &storage, 0, 71).is_empty());
let non_empty = prune_by_collect(&config, &storage, 0, 72);
assert!(!non_empty.is_empty());
storage.remove_from_storage(&non_empty);
assert!(storage.into_mdb().drain().is_empty());
assert!(prune_by_collect(&storage, 20, 30, 90).is_empty());
assert!(!storage.into_mdb().drain().is_empty());
// l1-digest is created every 2 blocks
// l2-digest is created every 4 blocks
// we want keep 1 additional changes tries
let config = Configuration { digest_interval: 2, digest_levels: 2 };
let storage = prepare_storage();
assert!(prune_by_collect(&config, &storage, 8, 69).is_empty());
assert!(prune_by_collect(&config, &storage, 8, 70).is_empty());
assert!(prune_by_collect(&config, &storage, 8, 71).is_empty());
assert!(prune_by_collect(&config, &storage, 8, 72).is_empty());
assert!(prune_by_collect(&config, &storage, 8, 73).is_empty());
assert!(prune_by_collect(&config, &storage, 8, 74).is_empty());
assert!(prune_by_collect(&config, &storage, 8, 75).is_empty());
let non_empty = prune_by_collect(&config, &storage, 8, 76);
assert!(!non_empty.is_empty());
storage.remove_from_storage(&non_empty);
assert!(storage.into_mdb().drain().is_empty());
let prune60_65 = prune_by_collect(&storage, 60, 65, 90);
assert!(!prune60_65.is_empty());
storage.remove_from_storage(&prune60_65);
assert!(!storage.into_mdb().drain().is_empty());
// l1-digest is created every 2 blocks
// we want keep 2 additional changes tries
let config = Configuration { digest_interval: 2, digest_levels: 1 };
let storage = prepare_storage();
assert!(prune_by_collect(&config, &storage, 4, 69).is_empty());
let non_empty = prune_by_collect(&config, &storage, 4, 70);
assert!(!non_empty.is_empty());
storage.remove_from_storage(&non_empty);
assert!(prune_by_collect(&config, &storage, 4, 71).is_empty());
let non_empty = prune_by_collect(&config, &storage, 4, 72);
assert!(!non_empty.is_empty());
storage.remove_from_storage(&non_empty);
let prune60_70 = prune_by_collect(&storage, 60, 70, 90);
assert!(!prune60_70.is_empty());
storage.remove_from_storage(&prune60_70);
assert!(storage.into_mdb().drain().is_empty());
}
#[test]
fn pruning_range_works() {
// DIGESTS ARE NOT CREATED + NO TRIES ARE PRUNED
assert_eq!(pruning_range(&config(10, 0), 2u64, 2u64), None);
// DIGESTS ARE NOT CREATED + SOME TRIES ARE PRUNED
assert_eq!(pruning_range(&config(10, 0), 100u64, 110u64), Some((10, 10)));
assert_eq!(pruning_range(&config(10, 0), 100u64, 210u64), Some((110, 110)));
// DIGESTS ARE CREATED + NO TRIES ARE PRUNED
assert_eq!(pruning_range(&config(10, 2), 2u64, 0u64), None);
assert_eq!(pruning_range(&config(10, 2), 30u64, 100u64), None);
assert_eq!(pruning_range(&config(::std::u32::MAX, 2), 1u64, 1024u64), None);
assert_eq!(pruning_range(&config(::std::u32::MAX, 2), ::std::u64::MAX, 1024u64), None);
assert_eq!(pruning_range(&config(32, 2), 2048u64, 512u64), None);
assert_eq!(pruning_range(&config(32, 2), 2048u64, 1024u64), None);
// DIGESTS ARE CREATED + SOME TRIES ARE PRUNED
// when we do not want to keep any highest-level-digests
// (system forces to keep at least one)
assert_eq!(pruning_range(&config(4, 2), 0u64, 32u64), Some((1, 16)));
assert_eq!(pruning_range(&config(4, 2), 0u64, 64u64), Some((33, 48)));
// when we want to keep 1 (last) highest-level-digest
assert_eq!(pruning_range(&config(4, 2), 16u64, 32u64), Some((1, 16)));
assert_eq!(pruning_range(&config(4, 2), 16u64, 64u64), Some((33, 48)));
// when we want to keep 1 (last) + 1 additional level digests
assert_eq!(pruning_range(&config(32, 2), 4096u64, 5120u64), Some((1, 1024)));
assert_eq!(pruning_range(&config(32, 2), 4096u64, 6144u64), Some((1025, 2048)));
}
#[test]
fn max_digest_intervals_to_keep_works() {
assert_eq!(max_digest_intervals_to_keep(1024u64, 1025), 1u64);
assert_eq!(max_digest_intervals_to_keep(1024u64, 1023), 1u64);
assert_eq!(max_digest_intervals_to_keep(1024u64, 512), 2u64);
assert_eq!(max_digest_intervals_to_keep(1024u64, 511), 2u64);
assert_eq!(max_digest_intervals_to_keep(1024u64, 100), 10u64);
}
#[test]
fn oldest_non_pruned_trie_works() {
// when digests are not created at all
assert_eq!(oldest_non_pruned_trie(&config(0, 0), 100u64, 10u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(0, 0), 100u64, 110u64), 11);
// when only l1 digests are created
assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100u64, 50u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100u64, 110u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100u64, 210u64), 101);
// when l2 digests are created
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 50u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 110u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 210u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 10110u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 20110u64), 10001);
}
}
substrate/primitives/state-machine/src/ext.rs
+28 -28
View File
@@ -18,7 +18,7 @@
use crate::{
backend::Backend, OverlayedChanges, StorageTransactionCache,
changes_trie::Storage as ChangesTrieStorage,
changes_trie::State as ChangesTrieState,
};
use hash_db::Hasher;
@@ -65,7 +65,7 @@ impl<B: error::Error, E: error::Error> error::Error for Error<B, E> {
}
/// Wraps a read-only backend, call executor, and current overlayed changes.
pub struct Ext<'a, H, N, B, T>
pub struct Ext<'a, H, N, B>
where
H: Hasher,
B: 'a + Backend<H>,
@@ -77,8 +77,8 @@ pub struct Ext<'a, H, N, B, T>
backend: &'a B,
/// The cache for the storage transactions.
storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H, N>,
/// Changes trie storage to read from.
changes_trie_storage: Option<&'a T>,
/// Changes trie state to read from.
changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
/// Pseudo-unique id used for tracing.
pub id: u16,
/// Dummy usage of N arg.
@@ -87,12 +87,11 @@ pub struct Ext<'a, H, N, B, T>
extensions: Option<&'a mut Extensions>,
}
impl<'a, H, N, B, T> Ext<'a, H, N, B, T>
impl<'a, H, N, B> Ext<'a, H, N, B>
where
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
B: 'a + Backend<H>,
T: 'a + ChangesTrieStorage<H, N>,
N: crate::changes_trie::BlockNumber,
{
@@ -101,13 +100,13 @@ where
overlay: &'a mut OverlayedChanges,
storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H, N>,
backend: &'a B,
changes_trie_storage: Option<&'a T>,
changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
extensions: Option<&'a mut Extensions>,
) -> Self {
Ext {
overlay,
backend,
changes_trie_storage,
changes_trie_state,
storage_transaction_cache,
id: rand::random(),
_phantom: Default::default(),
@@ -124,12 +123,11 @@ where
}
#[cfg(test)]
impl<'a, H, N, B, T> Ext<'a, H, N, B, T>
impl<'a, H, N, B> Ext<'a, H, N, B>
where
H: Hasher,
H::Out: Ord + 'static,
B: 'a + Backend<H>,
T: 'a + ChangesTrieStorage<H, N>,
N: crate::changes_trie::BlockNumber,
{
pub fn storage_pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
@@ -146,12 +144,11 @@ where
}
}
impl<'a, H, B, T, N> Externalities for Ext<'a, H, N, B, T>
impl<'a, H, B, N> Externalities for Ext<'a, H, N, B>
where
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
B: 'a + Backend<H>,
T: 'a + ChangesTrieStorage<H, N>,
N: crate::changes_trie::BlockNumber,
{
fn storage(&self, key: &[u8]) -> Option<Vec<u8>> {
@@ -564,7 +561,7 @@ where
let _guard = sp_panic_handler::AbortGuard::force_abort();
let root = self.overlay.changes_trie_root(
self.backend,
self.changes_trie_storage.clone(),
self.changes_trie_state.as_ref(),
Decode::decode(&mut &parent_hash[..]).map_err(|e|
trace!(
target: "state-trace",
@@ -586,11 +583,10 @@ where
}
}
impl<'a, H, B, T, N> sp_externalities::ExtensionStore for Ext<'a, H, N, B, T>
impl<'a, H, B, N> sp_externalities::ExtensionStore for Ext<'a, H, N, B>
where
H: Hasher,
B: 'a + Backend<H>,
T: 'a + ChangesTrieStorage<H, N>,
N: crate::changes_trie::BlockNumber,
{
fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any> {
@@ -602,19 +598,19 @@ where
mod tests {
use super::*;
use hex_literal::hex;
use num_traits::Zero;
use codec::Encode;
use sp_core::{H256, Blake2Hasher, storage::well_known_keys::EXTRINSIC_INDEX, map};
use crate::{
changes_trie::{
Configuration as ChangesTrieConfiguration,
InMemoryStorage as InMemoryChangesTrieStorage,
InMemoryStorage as TestChangesTrieStorage,
}, InMemoryBackend, overlayed_changes::OverlayedValue,
};
use sp_core::storage::{Storage, StorageChild};
type TestBackend = InMemoryBackend<Blake2Hasher>;
type TestChangesTrieStorage = InMemoryChangesTrieStorage<Blake2Hasher, u64>;
type TestExt<'a> = Ext<'a, Blake2Hasher, u64, TestBackend, TestChangesTrieStorage>;
type TestExt<'a> = Ext<'a, Blake2Hasher, u64, TestBackend>;
fn prepare_overlay_with_changes() -> OverlayedChanges {
OverlayedChanges {
@@ -629,10 +625,14 @@ mod tests {
}),
].into_iter().collect(),
committed: Default::default(),
changes_trie_config: Some(ChangesTrieConfiguration {
digest_interval: 0,
digest_levels: 0,
}),
collect_extrinsics: true,
}
}
fn changes_trie_config() -> ChangesTrieConfiguration {
ChangesTrieConfiguration {
digest_interval: 0,
digest_levels: 0,
}
}
@@ -646,13 +646,11 @@ mod tests {
}
#[test]
fn storage_changes_root_is_none_when_extrinsic_changes_are_none() {
fn storage_changes_root_is_none_when_state_is_not_provided() {
let mut overlay = prepare_overlay_with_changes();
let mut cache = StorageTransactionCache::default();
overlay.changes_trie_config = None;
let storage = TestChangesTrieStorage::with_blocks(vec![(100, Default::default())]);
let backend = TestBackend::default();
let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, Some(&storage), None);
let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None);
}
@@ -661,8 +659,9 @@ mod tests {
let mut overlay = prepare_overlay_with_changes();
let mut cache = StorageTransactionCache::default();
let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage));
let backend = TestBackend::default();
let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, Some(&storage), None);
let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None);
assert_eq!(
ext.storage_changes_root(&H256::default().encode()).unwrap(),
Some(hex!("bb0c2ef6e1d36d5490f9766cfcc7dfe2a6ca804504c3bb206053890d6dd02376").to_vec()),
@@ -675,8 +674,9 @@ mod tests {
let mut cache = StorageTransactionCache::default();
overlay.prospective.top.get_mut(&vec![1]).unwrap().value = None;
let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage));
let backend = TestBackend::default();
let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, Some(&storage), None);
let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None);
assert_eq!(
ext.storage_changes_root(&H256::default().encode()).unwrap(),
Some(hex!("96f5aae4690e7302737b6f9b7f8567d5bbb9eac1c315f80101235a92d9ec27f4").to_vec()),
substrate/primitives/state-machine/src/lib.rs
+36 -142
View File
@@ -23,8 +23,8 @@ use log::{warn, trace};
use hash_db::Hasher;
use codec::{Decode, Encode, Codec};
use sp_core::{
storage::{well_known_keys, ChildInfo}, NativeOrEncoded, NeverNativeValue,
traits::{CodeExecutor, CallInWasmExt}, hexdisplay::HexDisplay
storage::ChildInfo, NativeOrEncoded, NeverNativeValue,
traits::{CodeExecutor, CallInWasmExt}, hexdisplay::HexDisplay,
};
use overlayed_changes::OverlayedChangeSet;
use sp_externalities::Extensions;
@@ -49,15 +49,17 @@ pub use ext::Ext;
pub use backend::Backend;
pub use changes_trie::{
AnchorBlockId as ChangesTrieAnchorBlockId,
State as ChangesTrieState,
Storage as ChangesTrieStorage,
RootsStorage as ChangesTrieRootsStorage,
InMemoryStorage as InMemoryChangesTrieStorage,
BuildCache as ChangesTrieBuildCache,
CacheAction as ChangesTrieCacheAction,
ConfigurationRange as ChangesTrieConfigurationRange,
key_changes, key_changes_proof, key_changes_proof_check,
key_changes, key_changes_proof,
key_changes_proof_check, key_changes_proof_check_with_db,
prune as prune_changes_tries,
oldest_non_pruned_trie as oldest_non_pruned_changes_trie,
disabled_state as disabled_changes_trie_state,
BlockNumber as ChangesTrieBlockNumber,
};
pub use overlayed_changes::{OverlayedChanges, StorageChanges, StorageTransactionCache};
@@ -171,7 +173,7 @@ fn always_untrusted_wasm<E, R: Decode>() -> ExecutionManager<DefaultHandler<R, E
}
/// The substrate state machine.
pub struct StateMachine<'a, B, H, N, T, Exec>
pub struct StateMachine<'a, B, H, N, Exec>
where
H: Hasher,
B: Backend<H>,
@@ -183,23 +185,22 @@ pub struct StateMachine<'a, B, H, N, T, Exec>
call_data: &'a [u8],
overlay: &'a mut OverlayedChanges,
extensions: Extensions,
changes_trie_storage: Option<&'a T>,
changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
_marker: PhantomData<(H, N)>,
storage_transaction_cache: Option<&'a mut StorageTransactionCache<B::Transaction, H, N>>,
}
impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
Exec: CodeExecutor + Clone + 'static,
B: Backend<H>,
T: ChangesTrieStorage<H, N>,
N: crate::changes_trie::BlockNumber,
{
/// Creates new substrate state machine.
pub fn new(
backend: &'a B,
changes_trie_storage: Option<&'a T>,
changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
overlay: &'a mut OverlayedChanges,
exec: &'a Exec,
method: &'a str,
@@ -215,7 +216,7 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
call_data,
extensions,
overlay,
changes_trie_storage,
changes_trie_state,
_marker: PhantomData,
storage_transaction_cache: None,
}
@@ -273,7 +274,7 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
self.overlay,
cache,
self.backend,
self.changes_trie_storage.clone(),
self.changes_trie_state.clone(),
Some(&mut self.extensions),
);
@@ -388,20 +389,8 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
CallResult<R, Exec::Error>,
) -> CallResult<R, Exec::Error>
{
// read changes trie configuration. The reason why we're doing it here instead of the
// `OverlayedChanges` constructor is that we need proofs for this read as a part of
// proof-of-execution on light clients. And the proof is recorded by the backend which
// is created after OverlayedChanges
let init_overlay = |overlay: &mut OverlayedChanges, final_check: bool, backend: &B| {
let changes_trie_config = try_read_overlay_value(
overlay,
backend,
well_known_keys::CHANGES_TRIE_CONFIG
)?;
set_changes_trie_config(overlay, changes_trie_config, final_check)
};
init_overlay(self.overlay, false, &self.backend)?;
let changes_tries_enabled = self.changes_trie_state.is_some();
self.overlay.set_collect_extrinsics(changes_tries_enabled);
let result = {
let orig_prospective = self.overlay.prospective.clone();
@@ -433,16 +422,12 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
}
};
if result.is_ok() {
init_overlay(self.overlay, true, self.backend)?;
}
result.map_err(|e| Box::new(e) as _)
}
}
/// Prove execution using the given state backend, overlayed changes, and call executor.
pub fn prove_execution<B, H, Exec>(
pub fn prove_execution<B, H, N, Exec>(
mut backend: B,
overlay: &mut OverlayedChanges,
exec: &Exec,
@@ -454,10 +439,11 @@ where
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
Exec: CodeExecutor + Clone + 'static,
N: crate::changes_trie::BlockNumber,
{
let trie_backend = backend.as_trie_backend()
.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<dyn Error>)?;
prove_execution_on_trie_backend(trie_backend, overlay, exec, method, call_data)
prove_execution_on_trie_backend::<_, _, N, _>(trie_backend, overlay, exec, method, call_data)
}
/// Prove execution using the given trie backend, overlayed changes, and call executor.
@@ -469,7 +455,7 @@ where
///
/// Note: changes to code will be in place if this call is made again. For running partial
/// blocks (e.g. a transaction at a time), ensure a different method is used.
pub fn prove_execution_on_trie_backend<S, H, Exec>(
pub fn prove_execution_on_trie_backend<S, H, N, Exec>(
trie_backend: &TrieBackend<S, H>,
overlay: &mut OverlayedChanges,
exec: &Exec,
@@ -481,9 +467,10 @@ where
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
Exec: CodeExecutor + 'static + Clone,
N: crate::changes_trie::BlockNumber,
{
let proving_backend = proving_backend::ProvingBackend::new(trie_backend);
let mut sm = StateMachine::<_, H, _, InMemoryChangesTrieStorage<H, u64>, Exec>::new(
let mut sm = StateMachine::<_, H, N, Exec>::new(
&proving_backend, None, overlay, exec, method, call_data, Extensions::default(),
);
@@ -496,7 +483,7 @@ where
}
/// Check execution proof, generated by `prove_execution` call.
pub fn execution_proof_check<H, Exec>(
pub fn execution_proof_check<H, N, Exec>(
root: H::Out,
proof: StorageProof,
overlay: &mut OverlayedChanges,
@@ -508,13 +495,14 @@ where
H: Hasher,
Exec: CodeExecutor + Clone + 'static,
H::Out: Ord + 'static + codec::Codec,
N: crate::changes_trie::BlockNumber,
{
let trie_backend = create_proof_check_backend::<H>(root.into(), proof)?;
execution_proof_check_on_trie_backend(&trie_backend, overlay, exec, method, call_data)
execution_proof_check_on_trie_backend::<_, N, _>(&trie_backend, overlay, exec, method, call_data)
}
/// Check execution proof on proving backend, generated by `prove_execution` call.
pub fn execution_proof_check_on_trie_backend<H, Exec>(
pub fn execution_proof_check_on_trie_backend<H, N, Exec>(
trie_backend: &TrieBackend<MemoryDB<H>, H>,
overlay: &mut OverlayedChanges,
exec: &Exec,
@@ -525,8 +513,9 @@ where
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
Exec: CodeExecutor + Clone + 'static,
N: crate::changes_trie::BlockNumber,
{
let mut sm = StateMachine::<_, H, _, InMemoryChangesTrieStorage<H, u64>, Exec>::new(
let mut sm = StateMachine::<_, H, N, Exec>::new(
trie_backend, None, overlay, exec, method, call_data, Extensions::default(),
);
@@ -692,44 +681,6 @@ where
.map_err(|e| Box::new(e) as Box<dyn Error>)
}
/// Sets overlayed changes' changes trie configuration. Returns error if configuration
/// differs from previous OR config decode has failed.
fn set_changes_trie_config(
overlay: &mut OverlayedChanges,
config: Option<Vec<u8>>,
final_check: bool,
) -> Result<(), Box<dyn Error>> {
let config = match config {
Some(v) => Some(Decode::decode(&mut &v[..])
.map_err(|_| Box::new("Failed to decode changes trie configuration".to_owned()) as Box<dyn Error>)?),
None => None,
};
if final_check && overlay.changes_trie_config.is_some() != config.is_some() {
return Err(Box::new("Changes trie configuration change is not supported".to_owned()));
}
if let Some(config) = config {
if !overlay.set_changes_trie_config(config) {
return Err(Box::new("Changes trie configuration change is not supported".to_owned()));
}
}
Ok(())
}
/// Reads storage value from overlay or from the backend.
fn try_read_overlay_value<H, B>(
overlay: &OverlayedChanges,
backend: &B, key: &[u8],
) -> Result<Option<Vec<u8>>, Box<dyn Error>> where H: Hasher, B: Backend<H> {
match overlay.storage(key).map(|x| x.map(|x| x.to_vec())) {
Some(value) => Ok(value),
None => backend
.storage(key)
.map_err(|err| Box::new(ExecutionError::Backend(format!("{}", err))) as Box<dyn Error>),
}
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
@@ -737,10 +688,7 @@ mod tests {
use overlayed_changes::OverlayedValue;
use super::*;
use super::ext::Ext;
use super::changes_trie::{
InMemoryStorage as InMemoryChangesTrieStorage,
Configuration as ChangesTrieConfig,
};
use super::changes_trie::Configuration as ChangesTrieConfig;
use sp_core::{Blake2Hasher, map, traits::Externalities, storage::ChildStorageKey};
#[derive(Clone)]
@@ -770,7 +718,7 @@ mod tests {
) -> (CallResult<R, Self::Error>, bool) {
if self.change_changes_trie_config {
ext.place_storage(
well_known_keys::CHANGES_TRIE_CONFIG.to_vec(),
sp_core::storage::well_known_keys::CHANGES_TRIE_CONFIG.to_vec(),
Some(
ChangesTrieConfig {
digest_interval: 777,
@@ -816,11 +764,10 @@ mod tests {
fn execute_works() {
let backend = trie_backend::tests::test_trie();
let mut overlayed_changes = Default::default();
let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
let mut state_machine = StateMachine::new(
&backend,
Some(&changes_trie_storage),
changes_trie::disabled_state::<_, u64>(),
&mut overlayed_changes,
&DummyCodeExecutor {
change_changes_trie_config: false,
@@ -844,11 +791,10 @@ mod tests {
fn execute_works_with_native_else_wasm() {
let backend = trie_backend::tests::test_trie();
let mut overlayed_changes = Default::default();
let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
let mut state_machine = StateMachine::new(
&backend,
Some(&changes_trie_storage),
changes_trie::disabled_state::<_, u64>(),
&mut overlayed_changes,
&DummyCodeExecutor {
change_changes_trie_config: false,
@@ -869,11 +815,10 @@ mod tests {
let mut consensus_failed = false;
let backend = trie_backend::tests::test_trie();
let mut overlayed_changes = Default::default();
let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
let mut state_machine = StateMachine::new(
&backend,
Some(&changes_trie_storage),
changes_trie::disabled_state::<_, u64>(),
&mut overlayed_changes,
&DummyCodeExecutor {
change_changes_trie_config: false,
@@ -910,7 +855,7 @@ mod tests {
// fetch execution proof from 'remote' full node
let remote_backend = trie_backend::tests::test_trie();
let remote_root = remote_backend.storage_root(std::iter::empty()).0;
let (remote_result, remote_proof) = prove_execution(
let (remote_result, remote_proof) = prove_execution::<_, _, u64, _>(
remote_backend,
&mut Default::default(),
&executor,
@@ -919,7 +864,7 @@ mod tests {
).unwrap();
// check proof locally
let local_result = execution_proof_check::<Blake2Hasher, _>(
let local_result = execution_proof_check::<Blake2Hasher, u64, _>(
remote_root,
remote_proof,
&mut Default::default(),
@@ -956,13 +901,12 @@ mod tests {
};
{
let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
let mut cache = StorageTransactionCache::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
backend,
Some(&changes_trie_storage),
changes_trie::disabled_state::<_, u64>(),
None,
);
ext.clear_prefix(b"ab");
@@ -987,14 +931,13 @@ mod tests {
fn set_child_storage_works() {
let mut state = InMemoryBackend::<Blake2Hasher>::default();
let backend = state.as_trie_backend().unwrap();
let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
let mut overlay = OverlayedChanges::default();
let mut cache = StorageTransactionCache::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
backend,
Some(&changes_trie_storage),
changes_trie::disabled_state::<_, u64>(),
None,
);
@@ -1080,30 +1023,6 @@ mod tests {
);
}
#[test]
fn cannot_change_changes_trie_config() {
let backend = trie_backend::tests::test_trie();
let mut overlayed_changes = Default::default();
let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
let mut state_machine = StateMachine::new(
&backend,
Some(&changes_trie_storage),
&mut overlayed_changes,
&DummyCodeExecutor {
change_changes_trie_config: true,
native_available: false,
native_succeeds: true,
fallback_succeeds: true,
},
"test",
&[],
Default::default(),
);
assert!(state_machine.execute(ExecutionStrategy::NativeWhenPossible).is_err());
}
#[test]
fn child_storage_uuid() {
const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1");
@@ -1115,13 +1034,12 @@ mod tests {
let subtrie2 = ChildStorageKey::from_slice(b":child_storage:default:sub_test2").unwrap();
let mut transaction = {
let backend = test_trie();
let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
let mut cache = StorageTransactionCache::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
&backend,
Some(&changes_trie_storage),
changes_trie::disabled_state::<_, u64>(),
None,
);
ext.set_child_storage(subtrie1, CHILD_INFO_1, b"abc".to_vec(), b"def".to_vec());
@@ -1139,28 +1057,4 @@ mod tests {
}
assert!(!duplicate);
}
#[test]
fn cannot_change_changes_trie_config_with_native_else_wasm() {
let backend = trie_backend::tests::test_trie();
let mut overlayed_changes = Default::default();
let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
let mut state_machine = StateMachine::new(
&backend,
Some(&changes_trie_storage),
&mut overlayed_changes,
&DummyCodeExecutor {
change_changes_trie_config: true,
native_available: false,
native_succeeds: true,
fallback_succeeds: true,
},
"test",
&[],
Default::default(),
);
assert!(state_machine.execute(ExecutionStrategy::NativeElseWasm).is_err());
}
}
substrate/primitives/state-machine/src/overlayed_changes.rs
+17 -78
View File
@@ -19,8 +19,8 @@
use crate::{
backend::Backend, ChangesTrieTransaction,
changes_trie::{
NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig, BlockNumber, build_changes_trie,
Storage as ChangesTrieStorage,
NO_EXTRINSIC_INDEX, BlockNumber, build_changes_trie,
State as ChangesTrieState,
},
};
@@ -43,9 +43,8 @@ pub struct OverlayedChanges {
pub(crate) prospective: OverlayedChangeSet,
/// Committed changes.
pub(crate) committed: OverlayedChangeSet,
/// Changes trie configuration. None by default, but could be installed by the
/// runtime if it supports change tries.
pub(crate) changes_trie_config: Option<ChangesTrieConfig>,
/// True if extrinsiscs stats must be collected.
pub(crate) collect_extrinsics: bool,
}
/// The storage value, used inside OverlayedChanges.
@@ -184,20 +183,9 @@ impl OverlayedChanges {
self.prospective.is_empty() && self.committed.is_empty()
}
/// Sets the changes trie configuration.
///
/// Returns false if configuration has been set already and we now trying
/// to install different configuration. This isn't supported now.
pub(crate) fn set_changes_trie_config(&mut self, config: ChangesTrieConfig) -> bool {
if let Some(ref old_config) = self.changes_trie_config {
// we do not support changes trie configuration' change now
if *old_config != config {
return false;
}
}
self.changes_trie_config = Some(config);
true
/// Ask to collect/not to collect extrinsics indices where key(s) has been changed.
pub fn set_collect_extrinsics(&mut self, collect_extrinsics: bool) {
self.collect_extrinsics = collect_extrinsics;
}
/// Returns a double-Option: None if the key is unknown (i.e. and the query should be referred
@@ -442,11 +430,11 @@ impl OverlayedChanges {
/// Convert this instance with all changes into a [`StorageChanges`] instance.
pub fn into_storage_changes<
B: Backend<H>, H: Hasher, N: BlockNumber, T: ChangesTrieStorage<H, N>
B: Backend<H>, H: Hasher, N: BlockNumber
>(
self,
backend: &B,
changes_trie_storage: Option<&T>,
changes_trie_state: Option<&ChangesTrieState<H, N>>,
parent_hash: H::Out,
mut cache: StorageTransactionCache<B::Transaction, H, N>,
) -> Result<StorageChanges<B::Transaction, H, N>, String> where H::Out: Ord + Encode + 'static {
@@ -463,7 +451,7 @@ impl OverlayedChanges {
if cache.changes_trie_transaction.is_none() {
self.changes_trie_root(
backend,
changes_trie_storage,
changes_trie_state,
parent_hash,
false,
&mut cache,
@@ -501,7 +489,7 @@ impl OverlayedChanges {
/// Changes that are made outside of extrinsics, are marked with
/// `NO_EXTRINSIC_INDEX` index.
fn extrinsic_index(&self) -> Option<u32> {
match self.changes_trie_config.is_some() {
match self.collect_extrinsics {
true => Some(
self.storage(EXTRINSIC_INDEX)
.and_then(|idx| idx.and_then(|idx| Decode::decode(&mut &*idx).ok()))
@@ -557,17 +545,17 @@ impl OverlayedChanges {
/// # Panics
///
/// Panics on storage error, when `panic_on_storage_error` is set.
pub fn changes_trie_root<H: Hasher, N: BlockNumber, B: Backend<H>, T: ChangesTrieStorage<H, N>>(
pub fn changes_trie_root<'a, H: Hasher, N: BlockNumber, B: Backend<H>>(
&self,
backend: &B,
changes_trie_storage: Option<&T>,
changes_trie_state: Option<&'a ChangesTrieState<'a, H, N>>,
parent_hash: H::Out,
panic_on_storage_error: bool,
cache: &mut StorageTransactionCache<B::Transaction, H, N>,
) -> Result<Option<H::Out>, ()> where H::Out: Ord + Encode + 'static {
build_changes_trie::<_, T, H, N>(
build_changes_trie::<_, H, N>(
backend,
changes_trie_storage,
changes_trie_state,
self,
parent_hash,
panic_on_storage_error,
@@ -656,7 +644,6 @@ mod tests {
Blake2Hasher, traits::Externalities, storage::well_known_keys::EXTRINSIC_INDEX,
};
use crate::InMemoryBackend;
use crate::changes_trie::InMemoryStorage as InMemoryChangesTrieStorage;
use crate::ext::Ext;
use super::*;
@@ -718,13 +705,12 @@ mod tests {
..Default::default()
};
let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
let mut cache = StorageTransactionCache::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
&backend,
Some(&changes_trie_storage),
crate::changes_trie::disabled_state::<_, u64>(),
None,
);
const ROOT: [u8; 32] = hex!("39245109cef3758c2eed2ccba8d9b370a917850af3824bc8348d505df2c298fa");
@@ -732,57 +718,10 @@ mod tests {
assert_eq!(&ext.storage_root()[..], &ROOT);
}
#[test]
fn changes_trie_configuration_is_saved() {
let mut overlay = OverlayedChanges::default();
assert!(overlay.changes_trie_config.is_none());
assert_eq!(
overlay.set_changes_trie_config(
ChangesTrieConfig { digest_interval: 4, digest_levels: 1, },
),
true,
);
assert!(overlay.changes_trie_config.is_some());
}
#[test]
fn changes_trie_configuration_is_saved_twice() {
let mut overlay = OverlayedChanges::default();
assert!(overlay.changes_trie_config.is_none());
assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
digest_interval: 4, digest_levels: 1,
}), true);
overlay.set_extrinsic_index(0);
overlay.set_storage(vec![1], Some(vec![2]));
assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
digest_interval: 4, digest_levels: 1,
}), true);
assert_eq!(
strip_extrinsic_index(&overlay.prospective.top),
vec![
(vec![1], OverlayedValue { value: Some(vec![2]),
extrinsics: Some(vec![0].into_iter().collect()) }),
].into_iter().collect(),
);
}
#[test]
fn panics_when_trying_to_save_different_changes_trie_configuration() {
let mut overlay = OverlayedChanges::default();
assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
digest_interval: 4, digest_levels: 1,
}), true);
assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
digest_interval: 2, digest_levels: 1,
}), false);
}
#[test]
fn extrinsic_changes_are_collected() {
let mut overlay = OverlayedChanges::default();
let _ = overlay.set_changes_trie_config(ChangesTrieConfig {
digest_interval: 4, digest_levels: 1,
});
overlay.set_collect_extrinsics(true);
overlay.set_storage(vec![100], Some(vec![101]));
substrate/primitives/state-machine/src/testing.rs
+17 -8
View File
@@ -17,12 +17,15 @@
//! Test implementation for Externalities.
use std::any::{Any, TypeId};
use codec::Decode;
use hash_db::Hasher;
use crate::{
backend::Backend, OverlayedChanges, StorageTransactionCache, ext::Ext, InMemoryBackend,
changes_trie::{
Configuration as ChangesTrieConfiguration,
InMemoryStorage as ChangesTrieInMemoryStorage,
BlockNumber as ChangesTrieBlockNumber,
State as ChangesTrieState,
},
};
use sp_core::{
@@ -45,6 +48,7 @@ where
<InMemoryBackend<H> as Backend<H>>::Transaction, H, N
>,
backend: InMemoryBackend<H>,
changes_trie_config: Option<ChangesTrieConfiguration>,
changes_trie_storage: ChangesTrieInMemoryStorage<H, N>,
extensions: Extensions,
}
@@ -54,12 +58,19 @@ impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
H::Out: Ord + 'static + codec::Codec
{
/// Get externalities implementation.
pub fn ext(&mut self) -> Ext<H, N, InMemoryBackend<H>, ChangesTrieInMemoryStorage<H, N>> {
pub fn ext(&mut self) -> Ext<H, N, InMemoryBackend<H>> {
Ext::new(
&mut self.overlay,
&mut self.storage_transaction_cache,
&self.backend,
Some(&self.changes_trie_storage),
match self.changes_trie_config.clone() {
Some(config) => Some(ChangesTrieState {
config,
zero: 0.into(),
storage: &self.changes_trie_storage,
}),
None => None,
},
Some(&mut self.extensions),
)
}
@@ -72,21 +83,19 @@ impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
/// Create a new instance of `TestExternalities` with code and storage.
pub fn new_with_code(code: &[u8], mut storage: Storage) -> Self {
let mut overlay = OverlayedChanges::default();
let changes_trie_config = storage.top.get(CHANGES_TRIE_CONFIG)
.and_then(|v| Decode::decode(&mut &v[..]).ok());
overlay.set_collect_extrinsics(changes_trie_config.is_some());
assert!(storage.top.keys().all(|key| !is_child_storage_key(key)));
assert!(storage.children.keys().all(|key| is_child_storage_key(key)));
super::set_changes_trie_config(
&mut overlay,
storage.top.get(&CHANGES_TRIE_CONFIG.to_vec()).cloned(),
false,
).expect("changes trie configuration is correct in test env; qed");
storage.top.insert(HEAP_PAGES.to_vec(), 8u64.encode());
storage.top.insert(CODE.to_vec(), code.to_vec());
TestExternalities {
overlay,
changes_trie_config,
changes_trie_storage: ChangesTrieInMemoryStorage::new(),
backend: storage.into(),
extensions: Default::default(),
substrate/test-utils/runtime/client/src/block_builder_ext.rs
+13
View File
@@ -17,6 +17,7 @@
//! Block Builder extensions for tests.
use sp_api::{ApiExt, ProvideRuntimeApi};
use sp_core::ChangesTrieConfiguration;
use sc_client_api::backend;
use sp_runtime::traits::HasherFor;
@@ -32,6 +33,11 @@ pub trait BlockBuilderExt {
key: Vec<u8>,
value: Option<Vec<u8>>,
) -> Result<(), sp_blockchain::Error>;
/// Add changes trie configuration update extrinsic to the block.
fn push_changes_trie_configuration_update(
&mut self,
new_config: Option<ChangesTrieConfiguration>,
) -> Result<(), sp_blockchain::Error>;
}
impl<'a, A, B> BlockBuilderExt for sc_block_builder::BlockBuilder<'a, substrate_test_runtime::Block, A, B> where
@@ -57,4 +63,11 @@ impl<'a, A, B> BlockBuilderExt for sc_block_builder::BlockBuilder<'a, substrate_
) -> Result<(), sp_blockchain::Error> {
self.push(substrate_test_runtime::Extrinsic::StorageChange(key, value))
}
fn push_changes_trie_configuration_update(
&mut self,
new_config: Option<ChangesTrieConfiguration>,
) -> Result<(), sp_blockchain::Error> {
self.push(substrate_test_runtime::Extrinsic::ChangesTrieConfigUpdate(new_config))
}
}
substrate/test-utils/runtime/client/src/lib.rs
+6 -7
View File
@@ -30,7 +30,7 @@ pub use sc_client::LongestChain;
pub use self::block_builder_ext::BlockBuilderExt;
use sp_core::sr25519;
use sp_core::{sr25519, ChangesTrieConfiguration};
use sp_core::storage::{ChildInfo, Storage, StorageChild};
use substrate_test_runtime::genesismap::{GenesisConfig, additional_storage_with_genesis};
use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Hash as HashT, NumberFor, HasherFor};
@@ -42,7 +42,6 @@ use sc_client::{
},
};
/// A prelude to import in tests.
pub mod prelude {
// Trait extensions
@@ -92,7 +91,7 @@ pub type LightExecutor = sc_client::light::call_executor::GenesisCallExecutor<
/// Parameters of test-client builder with test-runtime.
#[derive(Default)]
pub struct GenesisParameters {
support_changes_trie: bool,
changes_trie_config: Option<ChangesTrieConfiguration>,
heap_pages_override: Option<u64>,
extra_storage: Storage,
}
@@ -100,7 +99,7 @@ pub struct GenesisParameters {
impl GenesisParameters {
fn genesis_config(&self) -> GenesisConfig {
GenesisConfig::new(
self.support_changes_trie,
self.changes_trie_config.clone(),
vec![
sr25519::Public::from(Sr25519Keyring::Alice).into(),
sr25519::Public::from(Sr25519Keyring::Bob).into(),
@@ -171,9 +170,9 @@ pub trait TestClientBuilderExt<B>: Sized {
/// Returns a mutable reference to the genesis parameters.
fn genesis_init_mut(&mut self) -> &mut GenesisParameters;
/// Enable or disable support for changes trie in genesis.
fn set_support_changes_trie(mut self, support_changes_trie: bool) -> Self {
self.genesis_init_mut().support_changes_trie = support_changes_trie;
/// Set changes trie configuration for genesis.
fn changes_trie_config(mut self, config: Option<ChangesTrieConfiguration>) -> Self {
self.genesis_init_mut().changes_trie_config = config;
self
}
substrate/test-utils/runtime/src/genesismap.rs
+2 -5
View File
@@ -36,7 +36,7 @@ pub struct GenesisConfig {
impl GenesisConfig {
pub fn new(
support_changes_trie: bool,
changes_trie_config: Option<ChangesTrieConfiguration>,
authorities: Vec<AuthorityId>,
endowed_accounts: Vec<AccountId>,
balance: u64,
@@ -44,10 +44,7 @@ impl GenesisConfig {
extra_storage: Storage,
) -> Self {
GenesisConfig {
changes_trie_config: match support_changes_trie {
true => Some(super::changes_trie_config()),
false => None,
},
changes_trie_config,
authorities: authorities.clone(),
balances: endowed_accounts.into_iter().map(|a| (a, balance)).collect(),
heap_pages_override,
substrate/test-utils/runtime/src/lib.rs
+4 -9
View File
@@ -25,7 +25,7 @@ pub mod system;
use sp_std::{prelude::*, marker::PhantomData};
use codec::{Encode, Decode, Input, Error};
use sp_core::{Blake2Hasher, OpaqueMetadata, RuntimeDebug};
use sp_core::{Blake2Hasher, OpaqueMetadata, RuntimeDebug, ChangesTrieConfiguration};
use sp_application_crypto::{ed25519, sr25519, RuntimeAppPublic};
use trie_db::{TrieMut, Trie};
use sp_trie::PrefixedMemoryDB;
@@ -111,6 +111,7 @@ pub enum Extrinsic {
Transfer(Transfer, AccountSignature),
IncludeData(Vec<u8>),
StorageChange(Vec<u8>, Option<Vec<u8>>),
ChangesTrieConfigUpdate(Option<ChangesTrieConfiguration>),
}
#[cfg(feature = "std")]
@@ -135,6 +136,8 @@ impl BlindCheckable for Extrinsic {
},
Extrinsic::IncludeData(_) => Err(InvalidTransaction::BadProof.into()),
Extrinsic::StorageChange(key, value) => Ok(Extrinsic::StorageChange(key, value)),
Extrinsic::ChangesTrieConfigUpdate(new_config) =>
Ok(Extrinsic::ChangesTrieConfigUpdate(new_config)),
}
}
}
@@ -196,14 +199,6 @@ pub fn run_tests(mut input: &[u8]) -> Vec<u8> {
[stxs.len() as u8].encode()
}
/// Changes trie configuration (optionally) used in tests.
pub fn changes_trie_config() -> sp_core::ChangesTrieConfiguration {
sp_core::ChangesTrieConfiguration {
digest_interval: 4,
digest_levels: 2,
}
}
/// A type that can not be decoded.
#[derive(PartialEq)]
pub struct DecodeFails<B: BlockT> {
substrate/test-utils/runtime/src/system.rs
+24 -1
View File
@@ -35,7 +35,7 @@ use frame_system::Trait;
use crate::{
AccountId, BlockNumber, Extrinsic, Transfer, H256 as Hash, Block, Header, Digest, AuthorityId
};
use sp_core::storage::well_known_keys;
use sp_core::{storage::well_known_keys, ChangesTrieConfiguration};
const NONCE_OF: &[u8] = b"nonce:";
const BALANCE_OF: &[u8] = b"balance:";
@@ -51,6 +51,7 @@ decl_storage! {
Number get(fn number): Option<BlockNumber>;
ParentHash get(fn parent_hash): Hash;
NewAuthorities get(fn new_authorities): Option<Vec<AuthorityId>>;
NewChangesTrieConfig get(fn new_changes_trie_config): Option<Option<ChangesTrieConfiguration>>;
StorageDigest get(fn storage_digest): Option<Digest>;
Authorities get(fn authorities) config(): Vec<AuthorityId>;
}
@@ -206,6 +207,8 @@ pub fn finalize_block() -> Header {
let mut digest = <StorageDigest>::take().expect("StorageDigest is set by `initialize_block`");
let o_new_authorities = <NewAuthorities>::take();
let new_changes_trie_config = <NewChangesTrieConfig>::take();
// This MUST come after all changes to storage are done. Otherwise we will fail the
// “Storage root does not match that calculated” assertion.
let storage_root = Hash::decode(&mut &storage_root()[..])
@@ -222,6 +225,12 @@ pub fn finalize_block() -> Header {
digest.push(generic::DigestItem::Consensus(*b"babe", new_authorities.encode()));
}
if let Some(new_config) = new_changes_trie_config {
digest.push(generic::DigestItem::ChangesTrieSignal(
generic::ChangesTrieSignal::NewConfiguration(new_config)
));
}
Header {
number,
extrinsics_root,
@@ -244,6 +253,8 @@ fn execute_transaction_backend(utx: &Extrinsic) -> ApplyExtrinsicResult {
Extrinsic::AuthoritiesChange(ref new_auth) => execute_new_authorities_backend(new_auth),
Extrinsic::IncludeData(_) => Ok(Ok(())),
Extrinsic::StorageChange(key, value) => execute_storage_change(key, value.as_ref().map(|v| &**v)),
Extrinsic::ChangesTrieConfigUpdate(ref new_config) =>
execute_changes_trie_config_update(new_config.clone()),
}
}
@@ -286,6 +297,18 @@ fn execute_storage_change(key: &[u8], value: Option<&[u8]>) -> ApplyExtrinsicRes
Ok(Ok(()))
}
fn execute_changes_trie_config_update(new_config: Option<ChangesTrieConfiguration>) -> ApplyExtrinsicResult {
match new_config.clone() {
Some(new_config) => storage::unhashed::put_raw(
well_known_keys::CHANGES_TRIE_CONFIG,
&new_config.encode(),
),
None => storage::unhashed::kill(well_known_keys::CHANGES_TRIE_CONFIG),
}
<NewChangesTrieConfig>::put(new_config);
Ok(Ok(()))
}
#[cfg(feature = "std")]
fn info_expect_equal_hash(given: &Hash, expected: &Hash) {
use sp_core::hexdisplay::HexDisplay;