pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-12 04:11:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Kill the light client, CHTs and change tries. (#10080)

Browse Source 
* Remove light client, change tries and CHTs

* Update tests

* fmt

* Restore changes_root

* Fixed benches

* Cargo fmt

* fmt

* fmt
This commit is contained in:
Arkadiy Paronyan
2021-11-12 14:15:01 +01:00
committed by GitHub
parent 112b7dac47
commit 4cbbf0cf43
141 changed files with 532 additions and 17807 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/primitives/state-machine/src/backend.rs
-26
View File
@@ -292,32 +292,6 @@ impl<H: Hasher, KF: sp_trie::KeyFunction<H>> Consolidate for sp_trie::GenericMem
}
}
/// Insert input pairs into memory db.
#[cfg(test)]
pub(crate) fn insert_into_memory_db<H, I>(
mdb: &mut sp_trie::MemoryDB<H>,
input: I,
) -> Option<H::Out>
where
H: Hasher,
I: IntoIterator<Item = (StorageKey, StorageValue)>,
{
use sp_trie::{trie_types::TrieDBMut, TrieMut};
let mut root = <H as Hasher>::Out::default();
{
let mut trie = TrieDBMut::<H>::new(mdb, &mut root);
for (key, value) in input {
if let Err(e) = trie.insert(&key, &value) {
log::warn!(target: "trie", "Failed to write to trie: {}", e);
return None
}
}
}
Some(root)
}
/// Wrapper to create a [`RuntimeCode`] from a type that implements [`Backend`].
#[cfg(feature = "std")]
pub struct BackendRuntimeCode<'a, B, H> {
substrate/primitives/state-machine/src/basic.rs
-4
View File
@@ -309,10 +309,6 @@ impl Externalities for BasicExternalities {
.encode()
}
fn storage_changes_root(&mut self, _parent: &[u8]) -> Result<Option<Vec<u8>>, ()> {
Ok(None)
}
fn storage_start_transaction(&mut self) {
unimplemented!("Transactions are not supported by BasicExternalities");
}
substrate/primitives/state-machine/src/changes_trie/build.rs
-1083
View File
File diff suppressed because it is too large Load Diff
substrate/primitives/state-machine/src/changes_trie/build_cache.rs
-278
View File
@@ -1,278 +0,0 @@
// This file is part of Substrate.
// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Changes tries build cache.
use std::collections::{HashMap, HashSet};
use crate::StorageKey;
use sp_core::storage::PrefixedStorageKey;
/// Changes trie build cache.
///
/// Helps to avoid read of changes tries from the database when digest trie
/// is built. It holds changed keys for every block (indexed by changes trie
/// root) that could be referenced by future digest items. For digest entries
/// it also holds keys covered by this digest. Entries for top level digests
/// are never created, because they'll never be used to build other digests.
///
/// Entries are pruned from the cache once digest block that is using this entry
/// is inserted (because digest block will includes all keys from this entry).
/// When there's a fork, entries are pruned when first changes trie is inserted.
pub struct BuildCache<H, N> {
/// Map of block (implies changes trie) number => changes trie root.
roots_by_number: HashMap<N, H>,
/// Map of changes trie root => set of storage keys that are in this trie.
/// The `Option<Vec<u8>>` in inner `HashMap` stands for the child storage key.
/// If it is `None`, then the `HashSet` contains keys changed in top-level storage.
/// If it is `Some`, then the `HashSet` contains keys changed in child storage, identified by
/// the key.
changed_keys: HashMap<H, HashMap<Option<PrefixedStorageKey>, HashSet<StorageKey>>>,
}
/// The action to perform when block-with-changes-trie is imported.
#[derive(Debug, PartialEq)]
pub enum CacheAction<H, N> {
/// Cache data that has been collected when CT has been built.
CacheBuildData(CachedBuildData<H, N>),
/// Clear cache from all existing entries.
Clear,
}
/// The data that has been cached during changes trie building.
#[derive(Debug, PartialEq)]
pub struct CachedBuildData<H, N> {
block: N,
trie_root: H,
digest_input_blocks: Vec<N>,
changed_keys: HashMap<Option<PrefixedStorageKey>, HashSet<StorageKey>>,
}
/// The action to perform when block-with-changes-trie is imported.
#[derive(Debug, PartialEq)]
pub(crate) enum IncompleteCacheAction<N> {
/// Cache data that has been collected when CT has been built.
CacheBuildData(IncompleteCachedBuildData<N>),
/// Clear cache from all existing entries.
Clear,
}
/// The data (without changes trie root) that has been cached during changes trie building.
#[derive(Debug, PartialEq)]
pub(crate) struct IncompleteCachedBuildData<N> {
digest_input_blocks: Vec<N>,
changed_keys: HashMap<Option<PrefixedStorageKey>, HashSet<StorageKey>>,
}
impl<H, N> BuildCache<H, N>
where
N: Eq + ::std::hash::Hash,
H: Eq + ::std::hash::Hash + Clone,
{
/// Create new changes trie build cache.
pub fn new() -> Self {
BuildCache { roots_by_number: HashMap::new(), changed_keys: HashMap::new() }
}
/// Get cached changed keys for changes trie with given root.
pub fn get(
&self,
root: &H,
) -> Option<&HashMap<Option<PrefixedStorageKey>, HashSet<StorageKey>>> {
self.changed_keys.get(&root)
}
/// Execute given functor with cached entry for given block.
/// Returns true if the functor has been called and false otherwise.
pub fn with_changed_keys(
&self,
root: &H,
functor: &mut dyn FnMut(&HashMap<Option<PrefixedStorageKey>, HashSet<StorageKey>>),
) -> bool {
match self.changed_keys.get(&root) {
Some(changed_keys) => {
functor(changed_keys);
true
},
None => false,
}
}
/// Insert data into cache.
pub fn perform(&mut self, action: CacheAction<H, N>) {
match action {
CacheAction::CacheBuildData(data) => {
self.roots_by_number.insert(data.block, data.trie_root.clone());
self.changed_keys.insert(data.trie_root, data.changed_keys);
for digest_input_block in data.digest_input_blocks {
let digest_input_block_hash = self.roots_by_number.remove(&digest_input_block);
if let Some(digest_input_block_hash) = digest_input_block_hash {
self.changed_keys.remove(&digest_input_block_hash);
}
}
},
CacheAction::Clear => {
self.roots_by_number.clear();
self.changed_keys.clear();
},
}
}
}
impl<N> IncompleteCacheAction<N> {
/// Returns true if we need to collect changed keys for this action.
pub fn collects_changed_keys(&self) -> bool {
match *self {
IncompleteCacheAction::CacheBuildData(_) => true,
IncompleteCacheAction::Clear => false,
}
}
/// Complete cache action with computed changes trie root.
pub(crate) fn complete<H: Clone>(self, block: N, trie_root: &H) -> CacheAction<H, N> {
match self {
IncompleteCacheAction::CacheBuildData(build_data) =>
CacheAction::CacheBuildData(build_data.complete(block, trie_root.clone())),
IncompleteCacheAction::Clear => CacheAction::Clear,
}
}
/// Set numbers of blocks that are superseded by this new entry.
///
/// If/when this build data is committed to the cache, entries for these blocks
/// will be removed from the cache.
pub(crate) fn set_digest_input_blocks(self, digest_input_blocks: Vec<N>) -> Self {
match self {
IncompleteCacheAction::CacheBuildData(build_data) =>
IncompleteCacheAction::CacheBuildData(
build_data.set_digest_input_blocks(digest_input_blocks),
),
IncompleteCacheAction::Clear => IncompleteCacheAction::Clear,
}
}
/// Insert changed keys of given storage into cached data.
pub(crate) fn insert(
self,
storage_key: Option<PrefixedStorageKey>,
changed_keys: HashSet<StorageKey>,
) -> Self {
match self {
IncompleteCacheAction::CacheBuildData(build_data) =>
IncompleteCacheAction::CacheBuildData(build_data.insert(storage_key, changed_keys)),
IncompleteCacheAction::Clear => IncompleteCacheAction::Clear,
}
}
}
impl<N> IncompleteCachedBuildData<N> {
/// Create new cached data.
pub(crate) fn new() -> Self {
IncompleteCachedBuildData { digest_input_blocks: Vec::new(), changed_keys: HashMap::new() }
}
fn complete<H>(self, block: N, trie_root: H) -> CachedBuildData<H, N> {
CachedBuildData {
block,
trie_root,
digest_input_blocks: self.digest_input_blocks,
changed_keys: self.changed_keys,
}
}
fn set_digest_input_blocks(mut self, digest_input_blocks: Vec<N>) -> Self {
self.digest_input_blocks = digest_input_blocks;
self
}
fn insert(
mut self,
storage_key: Option<PrefixedStorageKey>,
changed_keys: HashSet<StorageKey>,
) -> Self {
self.changed_keys.insert(storage_key, changed_keys);
self
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn updated_keys_are_stored_when_non_top_level_digest_is_built() {
let mut data = IncompleteCachedBuildData::<u32>::new();
data = data.insert(None, vec![vec![1]].into_iter().collect());
assert_eq!(data.changed_keys.len(), 1);
let mut cache = BuildCache::new();
cache.perform(CacheAction::CacheBuildData(data.complete(1, 1)));
assert_eq!(cache.changed_keys.len(), 1);
assert_eq!(
cache.get(&1).unwrap().clone(),
vec![(None, vec![vec![1]].into_iter().collect())].into_iter().collect(),
);
}
#[test]
fn obsolete_entries_are_purged_when_new_ct_is_built() {
let mut cache = BuildCache::<u32, u32>::new();
cache.perform(CacheAction::CacheBuildData(
IncompleteCachedBuildData::new()
.insert(None, vec![vec![1]].into_iter().collect())
.complete(1, 1),
));
cache.perform(CacheAction::CacheBuildData(
IncompleteCachedBuildData::new()
.insert(None, vec![vec![2]].into_iter().collect())
.complete(2, 2),
));
cache.perform(CacheAction::CacheBuildData(
IncompleteCachedBuildData::new()
.insert(None, vec![vec![3]].into_iter().collect())
.complete(3, 3),
));
assert_eq!(cache.changed_keys.len(), 3);
cache.perform(CacheAction::CacheBuildData(
IncompleteCachedBuildData::new()
.set_digest_input_blocks(vec![1, 2, 3])
.complete(4, 4),
));
assert_eq!(cache.changed_keys.len(), 1);
cache.perform(CacheAction::CacheBuildData(
IncompleteCachedBuildData::new()
.insert(None, vec![vec![8]].into_iter().collect())
.complete(8, 8),
));
cache.perform(CacheAction::CacheBuildData(
IncompleteCachedBuildData::new()
.insert(None, vec![vec![12]].into_iter().collect())
.complete(12, 12),
));
assert_eq!(cache.changed_keys.len(), 3);
cache.perform(CacheAction::Clear);
assert_eq!(cache.changed_keys.len(), 0);
}
}
substrate/primitives/state-machine/src/changes_trie/build_iterator.rs
-487
View File
@@ -1,487 +0,0 @@
// This file is part of Substrate.
// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Structures and functions to return blocks whose changes are to be included
//! in given block's changes trie.
use crate::changes_trie::{BlockNumber, ConfigurationRange};
use num_traits::Zero;
/// Returns iterator of OTHER blocks that are required for inclusion into
/// changes trie of given block. Blocks are guaranteed to be returned in
/// ascending order.
///
/// Skewed digest is built IF block >= config.end.
pub fn digest_build_iterator<'a, Number: BlockNumber>(
config: ConfigurationRange<'a, Number>,
block: Number,
) -> DigestBuildIterator<Number> {
// prepare digest build parameters
let (_, _, digest_step) = match config.config.digest_level_at_block(config.zero, block.clone())
{
Some((current_level, digest_interval, digest_step)) =>
(current_level, digest_interval, digest_step),
None => return DigestBuildIterator::empty(),
};
DigestBuildIterator::new(
block.clone(),
config.end.unwrap_or(block),
config.config.digest_interval,
digest_step,
)
}
/// Changes trie build iterator that returns numbers of OTHER blocks that are
/// required for inclusion into changes trie of given block.
#[derive(Debug)]
pub struct DigestBuildIterator<Number: BlockNumber> {
/// Block we're building changes trie for. It could (logically) be a post-end block if we are
/// creating skewed digest.
block: Number,
/// Block that is a last block where current configuration is active. We have never yet created
/// anything after this block => digest that we're creating can't reference any blocks that are
/// >= end.
end: Number,
/// Interval of L1 digest blocks.
digest_interval: u32,
/// Max step that could be used when digest is created.
max_step: u32,
// Mutable data below:
/// Step of current blocks range.
current_step: u32,
/// Reverse step of current blocks range.
current_step_reverse: u32,
/// Current blocks range.
current_range: Option<BlocksRange<Number>>,
/// Last block that we have returned.
last_block: Option<Number>,
}
impl<Number: BlockNumber> DigestBuildIterator<Number> {
/// Create new digest build iterator.
pub fn new(block: Number, end: Number, digest_interval: u32, max_step: u32) -> Self {
DigestBuildIterator {
block,
end,
digest_interval,
max_step,
current_step: max_step,
current_step_reverse: 0,
current_range: None,
last_block: None,
}
}
/// Create empty digest build iterator.
pub fn empty() -> Self {
Self::new(Zero::zero(), Zero::zero(), 0, 0)
}
}
impl<Number: BlockNumber> Iterator for DigestBuildIterator<Number> {
type Item = Number;
fn next(&mut self) -> Option<Self::Item> {
// when we're building skewed digest, we might want to skip some blocks if
// they're not covered by current configuration
loop {
if let Some(next) = self.current_range.as_mut().and_then(|iter| iter.next()) {
if next < self.end {
self.last_block = Some(next.clone());
return Some(next)
}
}
// we are safe to use non-checking mul/sub versions here because:
// DigestBuildIterator is created only by internal function that is checking
// that all multiplications/subtractions are safe within max_step limit
let next_step_reverse = if self.current_step_reverse == 0 {
1
} else {
self.current_step_reverse * self.digest_interval
};
if next_step_reverse > self.max_step {
return None
}
self.current_step_reverse = next_step_reverse;
self.current_range = Some(BlocksRange::new(
match self.last_block.clone() {
Some(last_block) => last_block + self.current_step.into(),
None =>
self.block.clone() -
(self.current_step * self.digest_interval - self.current_step).into(),
},
self.block.clone(),
self.current_step.into(),
));
self.current_step = self.current_step / self.digest_interval;
if self.current_step == 0 {
self.current_step = 1;
}
}
}
}
/// Blocks range iterator with builtin step_by support.
#[derive(Debug)]
struct BlocksRange<Number: BlockNumber> {
current: Number,
end: Number,
step: Number,
}
impl<Number: BlockNumber> BlocksRange<Number> {
pub fn new(begin: Number, end: Number, step: Number) -> Self {
BlocksRange { current: begin, end, step }
}
}
impl<Number: BlockNumber> Iterator for BlocksRange<Number> {
type Item = Number;
fn next(&mut self) -> Option<Self::Item> {
if self.current >= self.end {
return None
}
let current = Some(self.current.clone());
self.current += self.step.clone();
current
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::changes_trie::Configuration;
fn digest_build_iterator(
digest_interval: u32,
digest_levels: u32,
zero: u64,
block: u64,
end: Option<u64>,
) -> DigestBuildIterator<u64> {
super::digest_build_iterator(
ConfigurationRange {
config: &Configuration { digest_interval, digest_levels },
zero,
end,
},
block,
)
}
fn digest_build_iterator_basic(
digest_interval: u32,
digest_levels: u32,
zero: u64,
block: u64,
) -> (u64, u32, u32) {
let iter = digest_build_iterator(digest_interval, digest_levels, zero, block, None);
(iter.block, iter.digest_interval, iter.max_step)
}
fn digest_build_iterator_blocks(
digest_interval: u32,
digest_levels: u32,
zero: u64,
block: u64,
end: Option<u64>,
) -> Vec<u64> {
digest_build_iterator(digest_interval, digest_levels, zero, block, end).collect()
}
#[test]
fn suggest_digest_inclusion_returns_empty_iterator() {
fn test_with_zero(zero: u64) {
let empty = (0, 0, 0);
assert_eq!(digest_build_iterator_basic(4, 16, zero, zero + 0), empty, "block is 0");
assert_eq!(
digest_build_iterator_basic(0, 16, zero, zero + 64),
empty,
"digest_interval is 0"
);
assert_eq!(
digest_build_iterator_basic(1, 16, zero, zero + 64),
empty,
"digest_interval is 1"
);
assert_eq!(
digest_build_iterator_basic(4, 0, zero, zero + 64),
empty,
"digest_levels is 0"
);
assert_eq!(
digest_build_iterator_basic(4, 16, zero, zero + 1),
empty,
"digest is not required for this block",
);
assert_eq!(
digest_build_iterator_basic(4, 16, zero, zero + 2),
empty,
"digest is not required for this block",
);
assert_eq!(
digest_build_iterator_basic(4, 16, zero, zero + 15),
empty,
"digest is not required for this block",
);
assert_eq!(
digest_build_iterator_basic(4, 16, zero, zero + 17),
empty,
"digest is not required for this block",
);
assert_eq!(
digest_build_iterator_basic(::std::u32::MAX / 2 + 1, 16, zero, ::std::u64::MAX,),
empty,
"digest_interval * 2 is greater than u64::MAX"
);
}
test_with_zero(0);
test_with_zero(1);
test_with_zero(2);
test_with_zero(4);
test_with_zero(17);
}
#[test]
fn suggest_digest_inclusion_returns_level1_iterator() {
fn test_with_zero(zero: u64) {
assert_eq!(
digest_build_iterator_basic(16, 1, zero, zero + 16),
(zero + 16, 16, 1),
"!(block % interval) && first digest level == block",
);
assert_eq!(
digest_build_iterator_basic(16, 1, zero, zero + 256),
(zero + 256, 16, 1),
"!(block % interval^2), but there's only 1 digest level",
);
assert_eq!(
digest_build_iterator_basic(16, 2, zero, zero + 32),
(zero + 32, 16, 1),
"second level digest is not required for this block",
);
assert_eq!(
digest_build_iterator_basic(16, 3, zero, zero + 4080),
(zero + 4080, 16, 1),
"second && third level digest are not required for this block",
);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn suggest_digest_inclusion_returns_level2_iterator() {
fn test_with_zero(zero: u64) {
assert_eq!(
digest_build_iterator_basic(16, 2, zero, zero + 256),
(zero + 256, 16, 16),
"second level digest",
);
assert_eq!(
digest_build_iterator_basic(16, 2, zero, zero + 4096),
(zero + 4096, 16, 16),
"!(block % interval^3), but there's only 2 digest levels",
);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn suggest_digest_inclusion_returns_level3_iterator() {
fn test_with_zero(zero: u64) {
assert_eq!(
digest_build_iterator_basic(16, 3, zero, zero + 4096),
(zero + 4096, 16, 256),
"third level digest: beginning",
);
assert_eq!(
digest_build_iterator_basic(16, 3, zero, zero + 8192),
(zero + 8192, 16, 256),
"third level digest: next",
);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn digest_iterator_returns_level1_blocks() {
fn test_with_zero(zero: u64) {
assert_eq!(
digest_build_iterator_blocks(16, 1, zero, zero + 16, None),
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
.iter()
.map(|item| zero + item)
.collect::<Vec<_>>()
);
assert_eq!(
digest_build_iterator_blocks(16, 1, zero, zero + 256, None),
[241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255]
.iter()
.map(|item| zero + item)
.collect::<Vec<_>>()
);
assert_eq!(
digest_build_iterator_blocks(16, 2, zero, zero + 32, None),
[17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
.iter()
.map(|item| zero + item)
.collect::<Vec<_>>()
);
assert_eq!(
digest_build_iterator_blocks(16, 3, zero, zero + 4080, None),
[
4065, 4066, 4067, 4068, 4069, 4070, 4071, 4072, 4073, 4074, 4075, 4076, 4077,
4078, 4079
]
.iter()
.map(|item| zero + item)
.collect::<Vec<_>>()
);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn digest_iterator_returns_level1_and_level2_blocks() {
fn test_with_zero(zero: u64) {
assert_eq!(
digest_build_iterator_blocks(16, 2, zero, zero + 256, None),
[
// level2 points to previous 16-1 level1 digests:
16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240,
// level2 is a level1 digest of 16-1 previous blocks:
241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255,
]
.iter()
.map(|item| zero + item)
.collect::<Vec<_>>(),
);
assert_eq!(
digest_build_iterator_blocks(16, 2, zero, zero + 4096, None),
[
// level2 points to previous 16-1 level1 digests:
3856, 3872, 3888, 3904, 3920, 3936, 3952, 3968, 3984, 4000, 4016, 4032, 4048,
4064, 4080, // level2 is a level1 digest of 16-1 previous blocks:
4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093,
4094, 4095,
]
.iter()
.map(|item| zero + item)
.collect::<Vec<_>>(),
);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn digest_iterator_returns_level1_and_level2_and_level3_blocks() {
fn test_with_zero(zero: u64) {
assert_eq!(
digest_build_iterator_blocks(16, 3, zero, zero + 4096, None),
[
// level3 points to previous 16-1 level2 digests:
256, 512, 768, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584,
3840, // level3 points to previous 16-1 level1 digests:
3856, 3872, 3888, 3904, 3920, 3936, 3952, 3968, 3984, 4000, 4016, 4032, 4048,
4064, 4080, // level3 is a level1 digest of 16-1 previous blocks:
4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093,
4094, 4095,
]
.iter()
.map(|item| zero + item)
.collect::<Vec<_>>(),
);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn digest_iterator_returns_skewed_digest_blocks() {
fn test_with_zero(zero: u64) {
assert_eq!(
digest_build_iterator_blocks(16, 3, zero, zero + 4096, Some(zero + 1338)),
[
// level3 MUST point to previous 16-1 level2 digests, BUT there are only 5:
256, 512, 768, 1024, 1280,
// level3 MUST point to previous 16-1 level1 digests, BUT there are only 3:
1296, 1312, 1328,
// level3 MUST be a level1 digest of 16-1 previous blocks, BUT there are only
// 9:
1329, 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337,
]
.iter()
.map(|item| zero + item)
.collect::<Vec<_>>(),
);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn digest_iterator_returns_skewed_digest_blocks_skipping_level() {
fn test_with_zero(zero: u64) {
assert_eq!(
digest_build_iterator_blocks(16, 3, zero, zero + 4096, Some(zero + 1284)),
[
// level3 MUST point to previous 16-1 level2 digests, BUT there are only 5:
256, 512, 768, 1024, 1280,
// level3 MUST point to previous 16-1 level1 digests, BUT there are NO ANY
// L1-digests: level3 MUST be a level1 digest of 16-1 previous blocks, BUT
// there are only 3:
1281, 1282, 1283,
]
.iter()
.map(|item| zero + item)
.collect::<Vec<_>>(),
);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
}
substrate/primitives/state-machine/src/changes_trie/changes_iterator.rs
-748
View File
@@ -1,748 +0,0 @@
// This file is part of Substrate.
// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Functions + iterator that traverses changes tries and returns all
//! (block, extrinsic) pairs where given key has been changed.
use crate::{
changes_trie::{
input::{ChildIndex, DigestIndex, DigestIndexValue, ExtrinsicIndex, ExtrinsicIndexValue},
storage::{InMemoryStorage, TrieBackendAdapter},
surface_iterator::{surface_iterator, SurfaceIterator},
AnchorBlockId, BlockNumber, ConfigurationRange, RootsStorage, Storage,
},
proving_backend::ProvingBackendRecorder,
trie_backend_essence::TrieBackendEssence,
};
use codec::{Codec, Decode, Encode};
use hash_db::Hasher;
use num_traits::Zero;
use sp_core::storage::PrefixedStorageKey;
use sp_trie::Recorder;
use std::{cell::RefCell, collections::VecDeque};
/// Return changes of given key at given blocks range.
/// `max` is the number of best known block.
/// Changes are returned in descending order (i.e. last block comes first).
pub fn key_changes<'a, H: Hasher, Number: BlockNumber>(
config: ConfigurationRange<'a, Number>,
storage: &'a dyn Storage<H, Number>,
begin: Number,
end: &'a AnchorBlockId<H::Out, Number>,
max: Number,
storage_key: Option<&'a PrefixedStorageKey>,
key: &'a [u8],
) -> Result<DrilldownIterator<'a, H, Number>, String> {
// we can't query any roots before root
let max = std::cmp::min(max, end.number.clone());
Ok(DrilldownIterator {
essence: DrilldownIteratorEssence {
storage_key,
key,
roots_storage: storage.as_roots_storage(),
storage,
begin: begin.clone(),
end,
config: config.clone(),
surface: surface_iterator(config, max, begin, end.number.clone())?,
extrinsics: Default::default(),
blocks: Default::default(),
_hasher: ::std::marker::PhantomData::<H>::default(),
},
})
}
/// Returns proof of changes of given key at given blocks range.
/// `max` is the number of best known block.
pub fn key_changes_proof<'a, H: Hasher, Number: BlockNumber>(
config: ConfigurationRange<'a, Number>,
storage: &dyn Storage<H, Number>,
begin: Number,
end: &AnchorBlockId<H::Out, Number>,
max: Number,
storage_key: Option<&PrefixedStorageKey>,
key: &[u8],
) -> Result<Vec<Vec<u8>>, String>
where
H::Out: Codec,
{
// we can't query any roots before root
let max = std::cmp::min(max, end.number.clone());
let mut iter = ProvingDrilldownIterator {
essence: DrilldownIteratorEssence {
storage_key,
key,
roots_storage: storage.as_roots_storage(),
storage,
begin: begin.clone(),
end,
config: config.clone(),
surface: surface_iterator(config, max, begin, end.number.clone())?,
extrinsics: Default::default(),
blocks: Default::default(),
_hasher: ::std::marker::PhantomData::<H>::default(),
},
proof_recorder: Default::default(),
};
// iterate to collect proof
while let Some(item) = iter.next() {
item?;
}
Ok(iter.extract_proof())
}
/// Check key changes proof and return changes of the key at given blocks range.
/// `max` is the number of best known block.
/// Changes are returned in descending order (i.e. last block comes first).
pub fn key_changes_proof_check<'a, H: Hasher, Number: BlockNumber>(
config: ConfigurationRange<'a, Number>,
roots_storage: &dyn RootsStorage<H, Number>,
proof: Vec<Vec<u8>>,
begin: Number,
end: &AnchorBlockId<H::Out, Number>,
max: Number,
storage_key: Option<&PrefixedStorageKey>,
key: &[u8],
) -> Result<Vec<(Number, u32)>, String>
where
H::Out: Encode,
{
key_changes_proof_check_with_db(
config,
roots_storage,
&InMemoryStorage::with_proof(proof),
begin,
end,
max,
storage_key,
key,
)
}
/// Similar to the `key_changes_proof_check` function, but works with prepared proof storage.
pub fn key_changes_proof_check_with_db<'a, H: Hasher, Number: BlockNumber>(
config: ConfigurationRange<'a, Number>,
roots_storage: &dyn RootsStorage<H, Number>,
proof_db: &InMemoryStorage<H, Number>,
begin: Number,
end: &AnchorBlockId<H::Out, Number>,
max: Number,
storage_key: Option<&PrefixedStorageKey>,
key: &[u8],
) -> Result<Vec<(Number, u32)>, String>
where
H::Out: Encode,
{
// we can't query any roots before root
let max = std::cmp::min(max, end.number.clone());
DrilldownIterator {
essence: DrilldownIteratorEssence {
storage_key,
key,
roots_storage,
storage: proof_db,
begin: begin.clone(),
end,
config: config.clone(),
surface: surface_iterator(config, max, begin, end.number.clone())?,
extrinsics: Default::default(),
blocks: Default::default(),
_hasher: ::std::marker::PhantomData::<H>::default(),
},
}
.collect()
}
/// Drilldown iterator - receives 'digest points' from surface iterator and explores
/// every point until extrinsic is found.
pub struct DrilldownIteratorEssence<'a, H, Number>
where
H: Hasher,
Number: BlockNumber,
H::Out: 'a,
{
storage_key: Option<&'a PrefixedStorageKey>,
key: &'a [u8],
roots_storage: &'a dyn RootsStorage<H, Number>,
storage: &'a dyn Storage<H, Number>,
begin: Number,
end: &'a AnchorBlockId<H::Out, Number>,
config: ConfigurationRange<'a, Number>,
surface: SurfaceIterator<'a, Number>,
extrinsics: VecDeque<(Number, u32)>,
blocks: VecDeque<(Number, Option<u32>)>,
_hasher: ::std::marker::PhantomData<H>,
}
impl<'a, H, Number> DrilldownIteratorEssence<'a, H, Number>
where
H: Hasher,
Number: BlockNumber,
H::Out: 'a,
{
pub fn next<F>(&mut self, trie_reader: F) -> Option<Result<(Number, u32), String>>
where
F: FnMut(&dyn Storage<H, Number>, H::Out, &[u8]) -> Result<Option<Vec<u8>>, String>,
{
match self.do_next(trie_reader) {
Ok(Some(res)) => Some(Ok(res)),
Ok(None) => None,
Err(err) => Some(Err(err)),
}
}
fn do_next<F>(&mut self, mut trie_reader: F) -> Result<Option<(Number, u32)>, String>
where
F: FnMut(&dyn Storage<H, Number>, H::Out, &[u8]) -> Result<Option<Vec<u8>>, String>,
{
loop {
if let Some((block, extrinsic)) = self.extrinsics.pop_front() {
return Ok(Some((block, extrinsic)))
}
if let Some((block, level)) = self.blocks.pop_front() {
// not having a changes trie root is an error because:
// we never query roots for future blocks
// AND trie roots for old blocks are known (both on full + light node)
let trie_root =
self.roots_storage.root(&self.end, block.clone())?.ok_or_else(|| {
format!("Changes trie root for block {} is not found", block.clone())
})?;
let trie_root = if let Some(storage_key) = self.storage_key {
let child_key =
ChildIndex { block: block.clone(), storage_key: storage_key.clone() }
.encode();
if let Some(trie_root) = trie_reader(self.storage, trie_root, &child_key)?
.and_then(|v| <Vec<u8>>::decode(&mut &v[..]).ok())
.map(|v| {
let mut hash = H::Out::default();
hash.as_mut().copy_from_slice(&v[..]);
hash
}) {
trie_root
} else {
continue
}
} else {
trie_root
};
// only return extrinsics for blocks before self.max
// most of blocks will be filtered out before pushing to `self.blocks`
// here we just throwing away changes at digest blocks we're processing
debug_assert!(
block >= self.begin,
"We shall not touch digests earlier than a range' begin"
);
if block <= self.end.number {
let extrinsics_key =
ExtrinsicIndex { block: block.clone(), key: self.key.to_vec() }.encode();
let extrinsics = trie_reader(self.storage, trie_root, &extrinsics_key);
if let Some(extrinsics) = extrinsics? {
if let Ok(extrinsics) = ExtrinsicIndexValue::decode(&mut &extrinsics[..]) {
self.extrinsics
.extend(extrinsics.into_iter().rev().map(|e| (block.clone(), e)));
}
}
}
let blocks_key =
DigestIndex { block: block.clone(), key: self.key.to_vec() }.encode();
let blocks = trie_reader(self.storage, trie_root, &blocks_key);
if let Some(blocks) = blocks? {
if let Ok(blocks) = <DigestIndexValue<Number>>::decode(&mut &blocks[..]) {
// filter level0 blocks here because we tend to use digest blocks,
// AND digest block changes could also include changes for out-of-range
// blocks
let begin = self.begin.clone();
let end = self.end.number.clone();
let config = self.config.clone();
self.blocks.extend(
blocks
.into_iter()
.rev()
.filter(|b| {
level.map(|level| level > 1).unwrap_or(true) ||
(*b >= begin && *b <= end)
})
.map(|b| {
let prev_level =
level.map(|level| Some(level - 1)).unwrap_or_else(|| {
Some(
config
.config
.digest_level_at_block(
config.zero.clone(),
b.clone(),
)
.map(|(level, _, _)| level)
.unwrap_or_else(|| Zero::zero()),
)
});
(b, prev_level)
}),
);
}
}
continue
}
match self.surface.next() {
Some(Ok(block)) => self.blocks.push_back(block),
Some(Err(err)) => return Err(err),
None => return Ok(None),
}
}
}
}
/// Exploring drilldown operator.
pub struct DrilldownIterator<'a, H, Number>
where
Number: BlockNumber,
H: Hasher,
H::Out: 'a,
{
essence: DrilldownIteratorEssence<'a, H, Number>,
}
impl<'a, H: Hasher, Number: BlockNumber> Iterator for DrilldownIterator<'a, H, Number>
where
H::Out: Encode,
{
type Item = Result<(Number, u32), String>;
fn next(&mut self) -> Option<Self::Item> {
self.essence.next(|storage, root, key| {
TrieBackendEssence::<_, H>::new(TrieBackendAdapter::new(storage), root).storage(key)
})
}
}
/// Proving drilldown iterator.
struct ProvingDrilldownIterator<'a, H, Number>
where
Number: BlockNumber,
H: Hasher,
H::Out: 'a,
{
essence: DrilldownIteratorEssence<'a, H, Number>,
proof_recorder: RefCell<Recorder<H::Out>>,
}
impl<'a, H, Number> ProvingDrilldownIterator<'a, H, Number>
where
Number: BlockNumber,
H: Hasher,
H::Out: 'a,
{
/// Consume the iterator, extracting the gathered proof in lexicographical order
/// by value.
pub fn extract_proof(self) -> Vec<Vec<u8>> {
self.proof_recorder
.into_inner()
.drain()
.into_iter()
.map(|n| n.data.to_vec())
.collect()
}
}
impl<'a, H, Number> Iterator for ProvingDrilldownIterator<'a, H, Number>
where
Number: BlockNumber,
H: Hasher,
H::Out: 'a + Codec,
{
type Item = Result<(Number, u32), String>;
fn next(&mut self) -> Option<Self::Item> {
let proof_recorder = &mut *self
.proof_recorder
.try_borrow_mut()
.expect("only fails when already borrowed; storage() is non-reentrant; qed");
self.essence.next(|storage, root, key| {
ProvingBackendRecorder::<_, H> {
backend: &TrieBackendEssence::new(TrieBackendAdapter::new(storage), root),
proof_recorder,
}
.storage(key)
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::changes_trie::{input::InputPair, storage::InMemoryStorage, Configuration};
use sp_runtime::traits::BlakeTwo256;
use std::iter::FromIterator;
fn child_key() -> PrefixedStorageKey {
let child_info = sp_core::storage::ChildInfo::new_default(&b"1"[..]);
child_info.prefixed_storage_key()
}
fn prepare_for_drilldown() -> (Configuration, InMemoryStorage<BlakeTwo256, u64>) {
let config = Configuration { digest_interval: 4, digest_levels: 2 };
let backend = InMemoryStorage::with_inputs(
vec![
// digest: 1..4 => [(3, 0)]
(1, vec![]),
(2, vec![]),
(
3,
vec![InputPair::ExtrinsicIndex(
ExtrinsicIndex { block: 3, key: vec![42] },
vec![0],
)],
),
(4, vec![InputPair::DigestIndex(DigestIndex { block: 4, key: vec![42] }, vec![3])]),
// digest: 5..8 => [(6, 3), (8, 1+2)]
(5, vec![]),
(
6,
vec![InputPair::ExtrinsicIndex(
ExtrinsicIndex { block: 6, key: vec![42] },
vec![3],
)],
),
(7, vec![]),
(
8,
vec![
InputPair::ExtrinsicIndex(
ExtrinsicIndex { block: 8, key: vec![42] },
vec![1, 2],
),
InputPair::DigestIndex(DigestIndex { block: 8, key: vec![42] }, vec![6]),
],
),
// digest: 9..12 => []
(9, vec![]),
(10, vec![]),
(11, vec![]),
(12, vec![]),
// digest: 0..16 => [4, 8]
(13, vec![]),
(14, vec![]),
(15, vec![]),
(
16,
vec![InputPair::DigestIndex(
DigestIndex { block: 16, key: vec![42] },
vec![4, 8],
)],
),
],
vec![(
child_key(),
vec![
(
1,
vec![InputPair::ExtrinsicIndex(
ExtrinsicIndex { block: 1, key: vec![42] },
vec![0],
)],
),
(
2,
vec![InputPair::ExtrinsicIndex(
ExtrinsicIndex { block: 2, key: vec![42] },
vec![3],
)],
),
(
16,
vec![
InputPair::ExtrinsicIndex(
ExtrinsicIndex { block: 16, key: vec![42] },
vec![5],
),
InputPair::DigestIndex(
DigestIndex { block: 16, key: vec![42] },
vec![2],
),
],
),
],
)],
);
(config, backend)
}
fn configuration_range<'a>(
config: &'a Configuration,
zero: u64,
) -> ConfigurationRange<'a, u64> {
ConfigurationRange { config, zero, end: None }
}
#[test]
fn drilldown_iterator_works() {
let (config, storage) = prepare_for_drilldown();
let drilldown_result = key_changes::<BlakeTwo256, u64>(
configuration_range(&config, 0),
&storage,
1,
&AnchorBlockId { hash: Default::default(), number: 16 },
16,
None,
&[42],
)
.and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)]));
let drilldown_result = key_changes::<BlakeTwo256, u64>(
configuration_range(&config, 0),
&storage,
1,
&AnchorBlockId { hash: Default::default(), number: 2 },
4,
None,
&[42],
)
.and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![]));
let drilldown_result = key_changes::<BlakeTwo256, u64>(
configuration_range(&config, 0),
&storage,
1,
&AnchorBlockId { hash: Default::default(), number: 3 },
4,
None,
&[42],
)
.and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(3, 0)]));
let drilldown_result = key_changes::<BlakeTwo256, u64>(
configuration_range(&config, 0),
&storage,
1,
&AnchorBlockId { hash: Default::default(), number: 7 },
7,
None,
&[42],
)
.and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(6, 3), (3, 0)]));
let drilldown_result = key_changes::<BlakeTwo256, u64>(
configuration_range(&config, 0),
&storage,
7,
&AnchorBlockId { hash: Default::default(), number: 8 },
8,
None,
&[42],
)
.and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1)]));
let drilldown_result = key_changes::<BlakeTwo256, u64>(
configuration_range(&config, 0),
&storage,
5,
&AnchorBlockId { hash: Default::default(), number: 7 },
8,
None,
&[42],
)
.and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(6, 3)]));
}
#[test]
fn drilldown_iterator_fails_when_storage_fails() {
let (config, storage) = prepare_for_drilldown();
storage.clear_storage();
assert!(key_changes::<BlakeTwo256, u64>(
configuration_range(&config, 0),
&storage,
1,
&AnchorBlockId { hash: Default::default(), number: 100 },
1000,
None,
&[42],
)
.and_then(|i| i.collect::<Result<Vec<_>, _>>())
.is_err());
assert!(key_changes::<BlakeTwo256, u64>(
configuration_range(&config, 0),
&storage,
1,
&AnchorBlockId { hash: Default::default(), number: 100 },
1000,
Some(&child_key()),
&[42],
)
.and_then(|i| i.collect::<Result<Vec<_>, _>>())
.is_err());
}
#[test]
fn drilldown_iterator_fails_when_range_is_invalid() {
let (config, storage) = prepare_for_drilldown();
assert!(key_changes::<BlakeTwo256, u64>(
configuration_range(&config, 0),
&storage,
1,
&AnchorBlockId { hash: Default::default(), number: 100 },
50,
None,
&[42],
)
.is_err());
assert!(key_changes::<BlakeTwo256, u64>(
configuration_range(&config, 0),
&storage,
20,
&AnchorBlockId { hash: Default::default(), number: 10 },
100,
None,
&[42],
)
.is_err());
}
#[test]
fn proving_drilldown_iterator_works() {
// happens on remote full node:
// create drilldown iterator that records all trie nodes during drilldown
let (remote_config, remote_storage) = prepare_for_drilldown();
let remote_proof = key_changes_proof::<BlakeTwo256, u64>(
configuration_range(&remote_config, 0),
&remote_storage,
1,
&AnchorBlockId { hash: Default::default(), number: 16 },
16,
None,
&[42],
)
.unwrap();
let (remote_config, remote_storage) = prepare_for_drilldown();
let remote_proof_child = key_changes_proof::<BlakeTwo256, u64>(
configuration_range(&remote_config, 0),
&remote_storage,
1,
&AnchorBlockId { hash: Default::default(), number: 16 },
16,
Some(&child_key()),
&[42],
)
.unwrap();
// happens on local light node:
// create drilldown iterator that works the same, but only depends on trie
let (local_config, local_storage) = prepare_for_drilldown();
local_storage.clear_storage();
let local_result = key_changes_proof_check::<BlakeTwo256, u64>(
configuration_range(&local_config, 0),
&local_storage,
remote_proof,
1,
&AnchorBlockId { hash: Default::default(), number: 16 },
16,
None,
&[42],
);
let (local_config, local_storage) = prepare_for_drilldown();
local_storage.clear_storage();
let local_result_child = key_changes_proof_check::<BlakeTwo256, u64>(
configuration_range(&local_config, 0),
&local_storage,
remote_proof_child,
1,
&AnchorBlockId { hash: Default::default(), number: 16 },
16,
Some(&child_key()),
&[42],
);
// check that drilldown result is the same as if it was happening at the full node
assert_eq!(local_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)]));
assert_eq!(local_result_child, Ok(vec![(16, 5), (2, 3)]));
}
#[test]
fn drilldown_iterator_works_with_skewed_digest() {
let config = Configuration { digest_interval: 4, digest_levels: 3 };
let mut config_range = configuration_range(&config, 0);
config_range.end = Some(91);
// when 4^3 deactivates at block 91:
// last L3 digest has been created at block#64
// skewed digest covers:
// L2 digests at blocks: 80
// L1 digests at blocks: 84, 88
// regular blocks: 89, 90, 91
let mut input = (1u64..92u64).map(|b| (b, vec![])).collect::<Vec<_>>();
// changed at block#63 and covered by L3 digest at block#64
input[63 - 1]
.1
.push(InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 63, key: vec![42] }, vec![0]));
input[64 - 1]
.1
.push(InputPair::DigestIndex(DigestIndex { block: 64, key: vec![42] }, vec![63]));
// changed at block#79 and covered by L2 digest at block#80 + skewed digest at block#91
input[79 - 1]
.1
.push(InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 79, key: vec![42] }, vec![1]));
input[80 - 1]
.1
.push(InputPair::DigestIndex(DigestIndex { block: 80, key: vec![42] }, vec![79]));
input[91 - 1]
.1
.push(InputPair::DigestIndex(DigestIndex { block: 91, key: vec![42] }, vec![80]));
let storage = InMemoryStorage::with_inputs(input, vec![]);
let drilldown_result = key_changes::<BlakeTwo256, u64>(
config_range,
&storage,
1,
&AnchorBlockId { hash: Default::default(), number: 91 },
100_000u64,
None,
&[42],
)
.and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(79, 1), (63, 0)]));
}
}
substrate/primitives/state-machine/src/changes_trie/input.rs
-207
View File
@@ -1,207 +0,0 @@
// This file is part of Substrate.
// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Different types of changes trie input pairs.
use crate::{changes_trie::BlockNumber, StorageKey, StorageValue};
use codec::{Decode, Encode, Error, Input, Output};
use sp_core::storage::PrefixedStorageKey;
/// Key of { changed key => set of extrinsic indices } mapping.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ExtrinsicIndex<Number: BlockNumber> {
/// Block at which this key has been inserted in the trie.
pub block: Number,
/// Storage key this node is responsible for.
pub key: StorageKey,
}
/// Value of { changed key => set of extrinsic indices } mapping.
pub type ExtrinsicIndexValue = Vec<u32>;
/// Key of { changed key => block/digest block numbers } mapping.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DigestIndex<Number: BlockNumber> {
/// Block at which this key has been inserted in the trie.
pub block: Number,
/// Storage key this node is responsible for.
pub key: StorageKey,
}
/// Key of { childtrie key => Childchange trie } mapping.
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct ChildIndex<Number: BlockNumber> {
/// Block at which this key has been inserted in the trie.
pub block: Number,
/// Storage key this node is responsible for.
pub storage_key: PrefixedStorageKey,
}
/// Value of { changed key => block/digest block numbers } mapping.
pub type DigestIndexValue<Number> = Vec<Number>;
/// Value of { changed key => block/digest block numbers } mapping.
/// That is the root of the child change trie.
pub type ChildIndexValue = Vec<u8>;
/// Single input pair of changes trie.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum InputPair<Number: BlockNumber> {
/// Element of { key => set of extrinsics where key has been changed } element mapping.
ExtrinsicIndex(ExtrinsicIndex<Number>, ExtrinsicIndexValue),
/// Element of { key => set of blocks/digest blocks where key has been changed } element
/// mapping.
DigestIndex(DigestIndex<Number>, DigestIndexValue<Number>),
/// Element of { childtrie key => Childchange trie } where key has been changed } element
/// mapping.
ChildIndex(ChildIndex<Number>, ChildIndexValue),
}
/// Single input key of changes trie.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum InputKey<Number: BlockNumber> {
/// Key of { key => set of extrinsics where key has been changed } element mapping.
ExtrinsicIndex(ExtrinsicIndex<Number>),
/// Key of { key => set of blocks/digest blocks where key has been changed } element mapping.
DigestIndex(DigestIndex<Number>),
/// Key of { childtrie key => Childchange trie } where key has been changed } element mapping.
ChildIndex(ChildIndex<Number>),
}
impl<Number: BlockNumber> InputPair<Number> {
/// Extract storage key that this pair corresponds to.
pub fn key(&self) -> Option<&[u8]> {
match *self {
InputPair::ExtrinsicIndex(ref key, _) => Some(&key.key),
InputPair::DigestIndex(ref key, _) => Some(&key.key),
InputPair::ChildIndex(_, _) => None,
}
}
}
impl<Number: BlockNumber> Into<(StorageKey, StorageValue)> for InputPair<Number> {
fn into(self) -> (StorageKey, StorageValue) {
match self {
InputPair::ExtrinsicIndex(key, value) => (key.encode(), value.encode()),
InputPair::DigestIndex(key, value) => (key.encode(), value.encode()),
InputPair::ChildIndex(key, value) => (key.encode(), value.encode()),
}
}
}
impl<Number: BlockNumber> Into<InputKey<Number>> for InputPair<Number> {
fn into(self) -> InputKey<Number> {
match self {
InputPair::ExtrinsicIndex(key, _) => InputKey::ExtrinsicIndex(key),
InputPair::DigestIndex(key, _) => InputKey::DigestIndex(key),
InputPair::ChildIndex(key, _) => InputKey::ChildIndex(key),
}
}
}
impl<Number: BlockNumber> ExtrinsicIndex<Number> {
pub fn key_neutral_prefix(block: Number) -> Vec<u8> {
let mut prefix = vec![1];
prefix.extend(block.encode());
prefix
}
}
impl<Number: BlockNumber> Encode for ExtrinsicIndex<Number> {
fn encode_to<W: Output + ?Sized>(&self, dest: &mut W) {
dest.push_byte(1);
self.block.encode_to(dest);
self.key.encode_to(dest);
}
}
impl<Number: BlockNumber> codec::EncodeLike for ExtrinsicIndex<Number> {}
impl<Number: BlockNumber> DigestIndex<Number> {
pub fn key_neutral_prefix(block: Number) -> Vec<u8> {
let mut prefix = vec![2];
prefix.extend(block.encode());
prefix
}
}
impl<Number: BlockNumber> Encode for DigestIndex<Number> {
fn encode_to<W: Output + ?Sized>(&self, dest: &mut W) {
dest.push_byte(2);
self.block.encode_to(dest);
self.key.encode_to(dest);
}
}
impl<Number: BlockNumber> ChildIndex<Number> {
pub fn key_neutral_prefix(block: Number) -> Vec<u8> {
let mut prefix = vec![3];
prefix.extend(block.encode());
prefix
}
}
impl<Number: BlockNumber> Encode for ChildIndex<Number> {
fn encode_to<W: Output + ?Sized>(&self, dest: &mut W) {
dest.push_byte(3);
self.block.encode_to(dest);
self.storage_key.encode_to(dest);
}
}
impl<Number: BlockNumber> codec::EncodeLike for DigestIndex<Number> {}
impl<Number: BlockNumber> Decode for InputKey<Number> {
fn decode<I: Input>(input: &mut I) -> Result<Self, Error> {
match input.read_byte()? {
1 => Ok(InputKey::ExtrinsicIndex(ExtrinsicIndex {
block: Decode::decode(input)?,
key: Decode::decode(input)?,
})),
2 => Ok(InputKey::DigestIndex(DigestIndex {
block: Decode::decode(input)?,
key: Decode::decode(input)?,
})),
3 => Ok(InputKey::ChildIndex(ChildIndex {
block: Decode::decode(input)?,
storage_key: PrefixedStorageKey::new(Decode::decode(input)?),
})),
_ => Err("Invalid input key variant".into()),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn extrinsic_index_serialized_and_deserialized() {
let original = ExtrinsicIndex { block: 777u64, key: vec![42] };
let serialized = original.encode();
let deserialized: InputKey<u64> = Decode::decode(&mut &serialized[..]).unwrap();
assert_eq!(InputKey::ExtrinsicIndex(original), deserialized);
}
#[test]
fn digest_index_serialized_and_deserialized() {
let original = DigestIndex { block: 777u64, key: vec![42] };
let serialized = original.encode();
let deserialized: InputKey<u64> = Decode::decode(&mut &serialized[..]).unwrap();
assert_eq!(InputKey::DigestIndex(original), deserialized);
}
}
substrate/primitives/state-machine/src/changes_trie/mod.rs
-428
View File
@@ -1,428 +0,0 @@
// This file is part of Substrate.
// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Changes trie related structures and functions.
//!
//! Changes trie is a trie built of { storage key => extrinsics } pairs
//! at the end of each block. For every changed storage key it contains
//! a pair, mapping key to the set of extrinsics where it has been changed.
//!
//! Optionally, every N blocks, additional level1-digest nodes are appended
//! to the changes trie, containing pairs { storage key => blocks }. For every
//! storage key that has been changed in PREVIOUS N-1 blocks (except for genesis
//! block) it contains a pair, mapping this key to the set of blocks where it
//! has been changed.
//!
//! Optionally, every N^digest_level (where digest_level > 1) blocks, additional
//! digest_level digest is created. It is built out of pairs { storage key => digest
//! block }, containing entries for every storage key that has been changed in
//! the last N*digest_level-1 blocks (except for genesis block), mapping these keys
//! to the set of lower-level digest blocks.
//!
//! Changes trie configuration could change within a time. The range of blocks, where
//! configuration has been active, is given by two blocks: zero and end. Zero block is
//! the block where configuration has been set. But the first changes trie that uses
//! this configuration will be built at the block zero+1. If configuration deactivates
//! at some block, this will be the end block of the configuration. It is also the
//! zero block of the next configuration.
//!
//! If configuration has the end block, it also means that 'skewed digest' has/should
//! been built at that block. If this is the block where max-level digest should have
//! been created, than it is simply max-level digest of this configuration. Otherwise,
//! it is the digest that covers all blocks since last max-level digest block was
//! created.
//!
//! Changes trie only contains the top level storage changes. Sub-level changes
//! are propagated through its storage root on the top level storage.
mod build;
mod build_cache;
mod build_iterator;
mod changes_iterator;
mod input;
mod prune;
mod storage;
mod surface_iterator;
pub use self::{
build_cache::{BuildCache, CacheAction, CachedBuildData},
changes_iterator::{
key_changes, key_changes_proof, key_changes_proof_check, key_changes_proof_check_with_db,
},
prune::prune,
storage::InMemoryStorage,
};
use crate::{
backend::Backend,
changes_trie::{
build::prepare_input,
build_cache::{IncompleteCacheAction, IncompleteCachedBuildData},
},
overlayed_changes::OverlayedChanges,
StorageKey,
};
use codec::{Decode, Encode};
use hash_db::{Hasher, Prefix};
use num_traits::{One, Zero};
use sp_core::{self, storage::PrefixedStorageKey};
use sp_trie::{trie_types::TrieDBMut, DBValue, MemoryDB, TrieMut};
use std::{
collections::{HashMap, HashSet},
convert::TryInto,
};
/// Requirements for block number that can be used with changes tries.
pub trait BlockNumber:
Send
+ Sync
+ 'static
+ std::fmt::Display
+ Clone
+ From<u32>
+ TryInto<u32>
+ One
+ Zero
+ PartialEq
+ Ord
+ std::hash::Hash
+ std::ops::Add<Self, Output = Self>
+ ::std::ops::Sub<Self, Output = Self>
+ std::ops::Mul<Self, Output = Self>
+ ::std::ops::Div<Self, Output = Self>
+ std::ops::Rem<Self, Output = Self>
+ std::ops::AddAssign<Self>
+ num_traits::CheckedMul
+ num_traits::CheckedSub
+ Decode
+ Encode
{
}
impl<T> BlockNumber for T where
T: Send
+ Sync
+ 'static
+ std::fmt::Display
+ Clone
+ From<u32>
+ TryInto<u32>
+ One
+ Zero
+ PartialEq
+ Ord
+ std::hash::Hash
+ std::ops::Add<Self, Output = Self>
+ ::std::ops::Sub<Self, Output = Self>
+ std::ops::Mul<Self, Output = Self>
+ ::std::ops::Div<Self, Output = Self>
+ std::ops::Rem<Self, Output = Self>
+ std::ops::AddAssign<Self>
+ num_traits::CheckedMul
+ num_traits::CheckedSub
+ Decode
+ Encode
{
}
/// Block identifier that could be used to determine fork of this block.
#[derive(Debug)]
pub struct AnchorBlockId<Hash: std::fmt::Debug, Number: BlockNumber> {
/// Hash of this block.
pub hash: Hash,
/// Number of this block.
pub number: Number,
}
/// Changes tries state at some block.
pub struct State<'a, H, Number> {
/// Configuration that is active at given block.
pub config: Configuration,
/// Configuration activation block number. Zero if it is the first configuration on the chain,
/// or number of the block that have emit NewConfiguration signal (thus activating
/// configuration starting from the **next** block).
pub zero: Number,
/// Underlying changes tries storage reference.
pub storage: &'a dyn Storage<H, Number>,
}
/// Changes trie storage. Provides access to trie roots and trie nodes.
pub trait RootsStorage<H: Hasher, Number: BlockNumber>: Send + Sync {
/// Resolve hash of the block into anchor.
fn build_anchor(&self, hash: H::Out) -> Result<AnchorBlockId<H::Out, Number>, String>;
/// Get changes trie root for the block with given number which is an ancestor (or the block
/// itself) of the anchor_block (i.e. anchor_block.number >= block).
fn root(
&self,
anchor: &AnchorBlockId<H::Out, Number>,
block: Number,
) -> Result<Option<H::Out>, String>;
}
/// Changes trie storage. Provides access to trie roots and trie nodes.
pub trait Storage<H: Hasher, Number: BlockNumber>: RootsStorage<H, Number> {
/// Casts from self reference to RootsStorage reference.
fn as_roots_storage(&self) -> &dyn RootsStorage<H, Number>;
/// Execute given functor with cached entry for given trie root.
/// Returns true if the functor has been called (cache entry exists) and false otherwise.
fn with_cached_changed_keys(
&self,
root: &H::Out,
functor: &mut dyn FnMut(&HashMap<Option<PrefixedStorageKey>, HashSet<StorageKey>>),
) -> bool;
/// Get a trie node.
fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String>;
}
/// Changes trie storage -> trie backend essence adapter.
pub struct TrieBackendStorageAdapter<'a, H: Hasher, Number: BlockNumber>(
pub &'a dyn Storage<H, Number>,
);
impl<'a, H: Hasher, N: BlockNumber> crate::TrieBackendStorage<H>
for TrieBackendStorageAdapter<'a, H, N>
{
type Overlay = sp_trie::MemoryDB<H>;
fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
self.0.get(key, prefix)
}
}
/// Changes trie configuration.
pub type Configuration = sp_core::ChangesTrieConfiguration;
/// Blocks range where configuration has been constant.
#[derive(Clone)]
pub struct ConfigurationRange<'a, N> {
/// Active configuration.
pub config: &'a Configuration,
/// Zero block of this configuration. The configuration is active starting from the next block.
pub zero: N,
/// End block of this configuration. It is the last block where configuration has been active.
pub end: Option<N>,
}
impl<'a, H, Number> State<'a, H, Number> {
/// Create state with given config and storage.
pub fn new(config: Configuration, zero: Number, storage: &'a dyn Storage<H, Number>) -> Self {
Self { config, zero, storage }
}
}
impl<'a, H, Number: Clone> Clone for State<'a, H, Number> {
fn clone(&self) -> Self {
State { config: self.config.clone(), zero: self.zero.clone(), storage: self.storage }
}
}
/// Create state where changes tries are disabled.
pub fn disabled_state<'a, H, Number>() -> Option<State<'a, H, Number>> {
None
}
/// Compute the changes trie root and transaction for given block.
/// Returns Err(()) if unknown `parent_hash` has been passed.
/// Returns Ok(None) if there's no data to perform computation.
/// Panics if background storage returns an error OR if insert to MemoryDB fails.
pub fn build_changes_trie<'a, B: Backend<H>, H: Hasher, Number: BlockNumber>(
backend: &B,
state: Option<&'a State<'a, H, Number>>,
changes: &OverlayedChanges,
parent_hash: H::Out,
panic_on_storage_error: bool,
) -> Result<Option<(MemoryDB<H>, H::Out, CacheAction<H::Out, Number>)>, ()>
where
H::Out: Ord + 'static + Encode,
{
/// Panics when `res.is_err() && panic`, otherwise it returns `Err(())` on an error.
fn maybe_panic<R, E: std::fmt::Debug>(
res: std::result::Result<R, E>,
panic: bool,
) -> std::result::Result<R, ()> {
res.map(Ok).unwrap_or_else(|e| {
if panic {
panic!(
"changes trie: storage access is not allowed to fail within runtime: {:?}",
e
)
} else {
Err(())
}
})
}
// when storage isn't provided, changes tries aren't created
let state = match state {
Some(state) => state,
None => return Ok(None),
};
// build_anchor error should not be considered fatal
let parent = state.storage.build_anchor(parent_hash).map_err(|_| ())?;
let block = parent.number.clone() + One::one();
// prepare configuration range - we already know zero block. Current block may be the end block
// if configuration has been changed in this block
let is_config_changed =
match changes.storage(sp_core::storage::well_known_keys::CHANGES_TRIE_CONFIG) {
Some(Some(new_config)) => new_config != &state.config.encode()[..],
Some(None) => true,
None => false,
};
let config_range = ConfigurationRange {
config: &state.config,
zero: state.zero.clone(),
end: if is_config_changed { Some(block.clone()) } else { None },
};
// storage errors are considered fatal (similar to situations when runtime fetches values from
// storage)
let (input_pairs, child_input_pairs, digest_input_blocks) = maybe_panic(
prepare_input::<B, H, Number>(
backend,
state.storage,
config_range.clone(),
changes,
&parent,
),
panic_on_storage_error,
)?;
// prepare cached data
let mut cache_action = prepare_cached_build_data(config_range, block.clone());
let needs_changed_keys = cache_action.collects_changed_keys();
cache_action = cache_action.set_digest_input_blocks(digest_input_blocks);
let mut mdb = MemoryDB::default();
let mut child_roots = Vec::with_capacity(child_input_pairs.len());
for (child_index, input_pairs) in child_input_pairs {
let mut not_empty = false;
let mut root = Default::default();
{
let mut trie = TrieDBMut::<H>::new(&mut mdb, &mut root);
let mut storage_changed_keys = HashSet::new();
for input_pair in input_pairs {
if needs_changed_keys {
if let Some(key) = input_pair.key() {
storage_changed_keys.insert(key.to_vec());
}
}
let (key, value) = input_pair.into();
not_empty = true;
maybe_panic(trie.insert(&key, &value), panic_on_storage_error)?;
}
cache_action =
cache_action.insert(Some(child_index.storage_key.clone()), storage_changed_keys);
}
if not_empty {
child_roots.push(input::InputPair::ChildIndex(child_index, root.as_ref().to_vec()));
}
}
let mut root = Default::default();
{
let mut trie = TrieDBMut::<H>::new(&mut mdb, &mut root);
for (key, value) in child_roots.into_iter().map(Into::into) {
maybe_panic(trie.insert(&key, &value), panic_on_storage_error)?;
}
let mut storage_changed_keys = HashSet::new();
for input_pair in input_pairs {
if needs_changed_keys {
if let Some(key) = input_pair.key() {
storage_changed_keys.insert(key.to_vec());
}
}
let (key, value) = input_pair.into();
maybe_panic(trie.insert(&key, &value), panic_on_storage_error)?;
}
cache_action = cache_action.insert(None, storage_changed_keys);
}
let cache_action = cache_action.complete(block, &root);
Ok(Some((mdb, root, cache_action)))
}
/// Prepare empty cached build data for given block.
fn prepare_cached_build_data<Number: BlockNumber>(
config: ConfigurationRange<Number>,
block: Number,
) -> IncompleteCacheAction<Number> {
// when digests are not enabled in configuration, we do not need to cache anything
// because it'll never be used again for building other tries
// => let's clear the cache
if !config.config.is_digest_build_enabled() {
return IncompleteCacheAction::Clear
}
// when this is the last block where current configuration is active
// => let's clear the cache
if config.end.as_ref() == Some(&block) {
return IncompleteCacheAction::Clear
}
// we do not need to cache anything when top-level digest trie is created, because
// it'll never be used again for building other tries
// => let's clear the cache
match config.config.digest_level_at_block(config.zero.clone(), block) {
Some((digest_level, _, _)) if digest_level == config.config.digest_levels =>
IncompleteCacheAction::Clear,
_ => IncompleteCacheAction::CacheBuildData(IncompleteCachedBuildData::new()),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn cache_is_cleared_when_digests_are_disabled() {
let config = Configuration { digest_interval: 0, digest_levels: 0 };
let config_range = ConfigurationRange { zero: 0, end: None, config: &config };
assert_eq!(prepare_cached_build_data(config_range, 8u32), IncompleteCacheAction::Clear);
}
#[test]
fn build_data_is_cached_when_digests_are_enabled() {
let config = Configuration { digest_interval: 8, digest_levels: 2 };
let config_range = ConfigurationRange { zero: 0, end: None, config: &config };
assert!(prepare_cached_build_data(config_range.clone(), 4u32).collects_changed_keys());
assert!(prepare_cached_build_data(config_range.clone(), 7u32).collects_changed_keys());
assert!(prepare_cached_build_data(config_range, 8u32).collects_changed_keys());
}
#[test]
fn cache_is_cleared_when_digests_are_enabled_and_top_level_digest_is_built() {
let config = Configuration { digest_interval: 8, digest_levels: 2 };
let config_range = ConfigurationRange { zero: 0, end: None, config: &config };
assert_eq!(prepare_cached_build_data(config_range, 64u32), IncompleteCacheAction::Clear);
}
#[test]
fn cache_is_cleared_when_end_block_of_configuration_is_built() {
let config = Configuration { digest_interval: 8, digest_levels: 2 };
let config_range = ConfigurationRange { zero: 0, end: Some(4u32), config: &config };
assert_eq!(
prepare_cached_build_data(config_range.clone(), 4u32),
IncompleteCacheAction::Clear
);
}
}
substrate/primitives/state-machine/src/changes_trie/prune.rs
-204
View File
@@ -1,204 +0,0 @@
// This file is part of Substrate.
// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Changes trie pruning-related functions.
use crate::{
changes_trie::{
input::{ChildIndex, InputKey},
storage::TrieBackendAdapter,
AnchorBlockId, BlockNumber, Storage,
},
proving_backend::ProvingBackendRecorder,
trie_backend_essence::TrieBackendEssence,
};
use codec::{Codec, Decode};
use hash_db::Hasher;
use log::warn;
use num_traits::One;
use sp_trie::Recorder;
/// Prune obsolete changes tries. Pruning happens at the same block, where highest
/// level digest is created. Pruning guarantees to save changes tries for last
/// `min_blocks_to_keep` blocks. We only prune changes tries at `max_digest_interval`
/// ranges.
pub fn prune<H: Hasher, Number: BlockNumber, F: FnMut(H::Out)>(
storage: &dyn Storage<H, Number>,
first: Number,
last: Number,
current_block: &AnchorBlockId<H::Out, Number>,
mut remove_trie_node: F,
) where
H::Out: Codec,
{
// delete changes trie for every block in range
let mut block = first;
loop {
if block >= last.clone() + One::one() {
break
}
let prev_block = block.clone();
block += One::one();
let block = prev_block;
let root = match storage.root(current_block, block.clone()) {
Ok(Some(root)) => root,
Ok(None) => continue,
Err(error) => {
// try to delete other tries
warn!(target: "trie", "Failed to read changes trie root from DB: {}", error);
continue
},
};
let children_roots = {
let trie_storage = TrieBackendEssence::<_, H>::new(
crate::changes_trie::TrieBackendStorageAdapter(storage),
root,
);
let child_prefix = ChildIndex::key_neutral_prefix(block.clone());
let mut children_roots = Vec::new();
trie_storage.for_key_values_with_prefix(&child_prefix, |mut key, mut value| {
if let Ok(InputKey::ChildIndex::<Number>(_trie_key)) = Decode::decode(&mut key) {
if let Ok(value) = <Vec<u8>>::decode(&mut value) {
let mut trie_root = <H as Hasher>::Out::default();
trie_root.as_mut().copy_from_slice(&value[..]);
children_roots.push(trie_root);
}
}
});
children_roots
};
for root in children_roots.into_iter() {
prune_trie(storage, root, &mut remove_trie_node);
}
prune_trie(storage, root, &mut remove_trie_node);
}
}
// Prune a trie.
fn prune_trie<H: Hasher, Number: BlockNumber, F: FnMut(H::Out)>(
storage: &dyn Storage<H, Number>,
root: H::Out,
remove_trie_node: &mut F,
) where
H::Out: Codec,
{
// enumerate all changes trie' keys, recording all nodes that have been 'touched'
// (effectively - all changes trie nodes)
let mut proof_recorder: Recorder<H::Out> = Default::default();
{
let mut trie = ProvingBackendRecorder::<_, H> {
backend: &TrieBackendEssence::new(TrieBackendAdapter::new(storage), root),
proof_recorder: &mut proof_recorder,
};
trie.record_all_keys();
}
// all nodes of this changes trie should be pruned
remove_trie_node(root);
for node in proof_recorder.drain().into_iter().map(|n| n.hash) {
remove_trie_node(node);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{backend::insert_into_memory_db, changes_trie::storage::InMemoryStorage};
use codec::Encode;
use sp_core::H256;
use sp_runtime::traits::BlakeTwo256;
use sp_trie::MemoryDB;
use std::collections::HashSet;
fn prune_by_collect(
storage: &dyn Storage<BlakeTwo256, u64>,
first: u64,
last: u64,
current_block: u64,
) -> HashSet<H256> {
let mut pruned_trie_nodes = HashSet::new();
let anchor = AnchorBlockId { hash: Default::default(), number: current_block };
prune(storage, first, last, &anchor, |node| {
pruned_trie_nodes.insert(node);
});
pruned_trie_nodes
}
#[test]
fn prune_works() {
fn prepare_storage() -> InMemoryStorage<BlakeTwo256, u64> {
let child_info = sp_core::storage::ChildInfo::new_default(&b"1"[..]);
let child_key =
ChildIndex { block: 67u64, storage_key: child_info.prefixed_storage_key() }
.encode();
let mut mdb1 = MemoryDB::<BlakeTwo256>::default();
let root1 =
insert_into_memory_db::<BlakeTwo256, _>(&mut mdb1, vec![(vec![10], vec![20])])
.unwrap();
let mut mdb2 = MemoryDB::<BlakeTwo256>::default();
let root2 = insert_into_memory_db::<BlakeTwo256, _>(
&mut mdb2,
vec![(vec![11], vec![21]), (vec![12], vec![22])],
)
.unwrap();
let mut mdb3 = MemoryDB::<BlakeTwo256>::default();
let ch_root3 =
insert_into_memory_db::<BlakeTwo256, _>(&mut mdb3, vec![(vec![110], vec![120])])
.unwrap();
let root3 = insert_into_memory_db::<BlakeTwo256, _>(
&mut mdb3,
vec![
(vec![13], vec![23]),
(vec![14], vec![24]),
(child_key, ch_root3.as_ref().encode()),
],
)
.unwrap();
let mut mdb4 = MemoryDB::<BlakeTwo256>::default();
let root4 =
insert_into_memory_db::<BlakeTwo256, _>(&mut mdb4, vec![(vec![15], vec![25])])
.unwrap();
let storage = InMemoryStorage::new();
storage.insert(65, root1, mdb1);
storage.insert(66, root2, mdb2);
storage.insert(67, root3, mdb3);
storage.insert(68, root4, mdb4);
storage
}
let storage = prepare_storage();
assert!(prune_by_collect(&storage, 20, 30, 90).is_empty());
assert!(!storage.into_mdb().drain().is_empty());
let storage = prepare_storage();
let prune60_65 = prune_by_collect(&storage, 60, 65, 90);
assert!(!prune60_65.is_empty());
storage.remove_from_storage(&prune60_65);
assert!(!storage.into_mdb().drain().is_empty());
let storage = prepare_storage();
let prune60_70 = prune_by_collect(&storage, 60, 70, 90);
assert!(!prune60_70.is_empty());
storage.remove_from_storage(&prune60_70);
assert!(storage.into_mdb().drain().is_empty());
}
}
substrate/primitives/state-machine/src/changes_trie/storage.rs
-214
View File
@@ -1,214 +0,0 @@
// This file is part of Substrate.
// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Changes trie storage utilities.
use crate::{
changes_trie::{AnchorBlockId, BlockNumber, BuildCache, RootsStorage, Storage},
trie_backend_essence::TrieBackendStorage,
StorageKey,
};
use hash_db::{Hasher, Prefix, EMPTY_PREFIX};
use parking_lot::RwLock;
use sp_core::storage::PrefixedStorageKey;
use sp_trie::{DBValue, MemoryDB};
use std::collections::{BTreeMap, HashMap, HashSet};
#[cfg(test)]
use crate::backend::insert_into_memory_db;
#[cfg(test)]
use crate::changes_trie::input::{ChildIndex, InputPair};
/// In-memory implementation of changes trie storage.
pub struct InMemoryStorage<H: Hasher, Number: BlockNumber> {
data: RwLock<InMemoryStorageData<H, Number>>,
cache: BuildCache<H::Out, Number>,
}
/// Adapter for using changes trie storage as a TrieBackendEssence' storage.
pub struct TrieBackendAdapter<'a, H: Hasher, Number: BlockNumber> {
storage: &'a dyn Storage<H, Number>,
_hasher: std::marker::PhantomData<(H, Number)>,
}
struct InMemoryStorageData<H: Hasher, Number: BlockNumber> {
roots: BTreeMap<Number, H::Out>,
mdb: MemoryDB<H>,
}
impl<H: Hasher, Number: BlockNumber> InMemoryStorage<H, Number> {
/// Creates storage from given in-memory database.
pub fn with_db(mdb: MemoryDB<H>) -> Self {
Self {
data: RwLock::new(InMemoryStorageData { roots: BTreeMap::new(), mdb }),
cache: BuildCache::new(),
}
}
/// Creates storage with empty database.
pub fn new() -> Self {
Self::with_db(Default::default())
}
/// Creates storage with given proof.
pub fn with_proof(proof: Vec<Vec<u8>>) -> Self {
use hash_db::HashDB;
let mut proof_db = MemoryDB::<H>::default();
for item in proof {
proof_db.insert(EMPTY_PREFIX, &item);
}
Self::with_db(proof_db)
}
/// Get mutable cache reference.
pub fn cache_mut(&mut self) -> &mut BuildCache<H::Out, Number> {
&mut self.cache
}
/// Create the storage with given blocks.
pub fn with_blocks(blocks: Vec<(Number, H::Out)>) -> Self {
Self {
data: RwLock::new(InMemoryStorageData {
roots: blocks.into_iter().collect(),
mdb: MemoryDB::default(),
}),
cache: BuildCache::new(),
}
}
#[cfg(test)]
pub fn with_inputs(
mut top_inputs: Vec<(Number, Vec<InputPair<Number>>)>,
children_inputs: Vec<(PrefixedStorageKey, Vec<(Number, Vec<InputPair<Number>>)>)>,
) -> Self {
let mut mdb = MemoryDB::default();
let mut roots = BTreeMap::new();
for (storage_key, child_input) in children_inputs {
for (block, pairs) in child_input {
let root =
insert_into_memory_db::<H, _>(&mut mdb, pairs.into_iter().map(Into::into));
if let Some(root) = root {
let ix = if let Some(ix) = top_inputs.iter().position(|v| v.0 == block) {
ix
} else {
top_inputs.push((block.clone(), Default::default()));
top_inputs.len() - 1
};
top_inputs[ix].1.push(InputPair::ChildIndex(
ChildIndex { block: block.clone(), storage_key: storage_key.clone() },
root.as_ref().to_vec(),
));
}
}
}
for (block, pairs) in top_inputs {
let root = insert_into_memory_db::<H, _>(&mut mdb, pairs.into_iter().map(Into::into));
if let Some(root) = root {
roots.insert(block, root);
}
}
InMemoryStorage {
data: RwLock::new(InMemoryStorageData { roots, mdb }),
cache: BuildCache::new(),
}
}
#[cfg(test)]
pub fn clear_storage(&self) {
self.data.write().mdb = MemoryDB::default(); // use new to be more correct
}
#[cfg(test)]
pub fn remove_from_storage(&self, keys: &HashSet<H::Out>) {
let mut data = self.data.write();
for key in keys {
data.mdb.remove_and_purge(key, hash_db::EMPTY_PREFIX);
}
}
#[cfg(test)]
pub fn into_mdb(self) -> MemoryDB<H> {
self.data.into_inner().mdb
}
/// Insert changes trie for given block.
pub fn insert(&self, block: Number, changes_trie_root: H::Out, trie: MemoryDB<H>) {
let mut data = self.data.write();
data.roots.insert(block, changes_trie_root);
data.mdb.consolidate(trie);
}
}
impl<H: Hasher, Number: BlockNumber> RootsStorage<H, Number> for InMemoryStorage<H, Number> {
fn build_anchor(&self, parent_hash: H::Out) -> Result<AnchorBlockId<H::Out, Number>, String> {
self.data
.read()
.roots
.iter()
.find(|(_, v)| **v == parent_hash)
.map(|(k, _)| AnchorBlockId { hash: parent_hash, number: k.clone() })
.ok_or_else(|| format!("Can't find associated number for block {:?}", parent_hash))
}
fn root(
&self,
_anchor_block: &AnchorBlockId<H::Out, Number>,
block: Number,
) -> Result<Option<H::Out>, String> {
Ok(self.data.read().roots.get(&block).cloned())
}
}
impl<H: Hasher, Number: BlockNumber> Storage<H, Number> for InMemoryStorage<H, Number> {
fn as_roots_storage(&self) -> &dyn RootsStorage<H, Number> {
self
}
fn with_cached_changed_keys(
&self,
root: &H::Out,
functor: &mut dyn FnMut(&HashMap<Option<PrefixedStorageKey>, HashSet<StorageKey>>),
) -> bool {
self.cache.with_changed_keys(root, functor)
}
fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
MemoryDB::<H>::get(&self.data.read().mdb, key, prefix)
}
}
impl<'a, H: Hasher, Number: BlockNumber> TrieBackendAdapter<'a, H, Number> {
pub fn new(storage: &'a dyn Storage<H, Number>) -> Self {
Self { storage, _hasher: Default::default() }
}
}
impl<'a, H, Number> TrieBackendStorage<H> for TrieBackendAdapter<'a, H, Number>
where
Number: BlockNumber,
H: Hasher,
{
type Overlay = MemoryDB<H>;
fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
self.storage.get(key, prefix)
}
}
substrate/primitives/state-machine/src/changes_trie/surface_iterator.rs
-326
View File
@@ -1,326 +0,0 @@
// This file is part of Substrate.
// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The best way to understand how this iterator works is to imagine some 2D terrain that have some
//! mountains (digest changes tries) and valleys (changes tries for regular blocks). There are gems
//! (blocks) beneath the terrain. Given the request to find all gems in the range [X1; X2] this
//! iterator will return **minimal set** of points at the terrain (mountains and valleys) inside
//! this range that have to be drilled down to search for gems.
use crate::changes_trie::{BlockNumber, ConfigurationRange};
use num_traits::One;
/// Returns surface iterator for given range of blocks.
///
/// `max` is the number of best block, known to caller. We can't access any changes tries
/// that are built after this block, even though we may have them built already.
pub fn surface_iterator<'a, Number: BlockNumber>(
config: ConfigurationRange<'a, Number>,
max: Number,
begin: Number,
end: Number,
) -> Result<SurfaceIterator<'a, Number>, String> {
let (current, current_begin, digest_step, digest_level) =
lower_bound_max_digest(config.clone(), max.clone(), begin.clone(), end)?;
Ok(SurfaceIterator {
config,
begin,
max,
current: Some(current),
current_begin,
digest_step,
digest_level,
})
}
/// Surface iterator - only traverses top-level digests from given range and tries to find
/// all valid digest changes.
///
/// Iterator item is the tuple of (last block of the current point + digest level of the current
/// point). Digest level is Some(0) when it is regular block, is Some(non-zero) when it is digest
/// block and None if it is skewed digest block.
pub struct SurfaceIterator<'a, Number: BlockNumber> {
config: ConfigurationRange<'a, Number>,
begin: Number,
max: Number,
current: Option<Number>,
current_begin: Number,
digest_step: u32,
digest_level: Option<u32>,
}
impl<'a, Number: BlockNumber> Iterator for SurfaceIterator<'a, Number> {
type Item = Result<(Number, Option<u32>), String>;
fn next(&mut self) -> Option<Self::Item> {
let current = self.current.clone()?;
let digest_level = self.digest_level;
if current < self.digest_step.into() {
self.current = None;
} else {
let next = current.clone() - self.digest_step.into();
if next.is_zero() || next < self.begin {
self.current = None;
} else if next > self.current_begin {
self.current = Some(next);
} else {
let max_digest_interval = lower_bound_max_digest(
self.config.clone(),
self.max.clone(),
self.begin.clone(),
next,
);
let (current, current_begin, digest_step, digest_level) = match max_digest_interval
{
Err(err) => return Some(Err(err)),
Ok(range) => range,
};
self.current = Some(current);
self.current_begin = current_begin;
self.digest_step = digest_step;
self.digest_level = digest_level;
}
}
Some(Ok((current, digest_level)))
}
}
/// Returns parameters of highest level digest block that includes the end of given range
/// and tends to include the whole range.
fn lower_bound_max_digest<'a, Number: BlockNumber>(
config: ConfigurationRange<'a, Number>,
max: Number,
begin: Number,
end: Number,
) -> Result<(Number, Number, u32, Option<u32>), String> {
if end > max || begin > end {
return Err(format!("invalid changes range: {}..{}/{}", begin, end, max))
}
if begin <= config.zero ||
config.end.as_ref().map(|config_end| end > *config_end).unwrap_or(false)
{
return Err(format!(
"changes trie range is not covered by configuration: {}..{}/{}..{}",
begin,
end,
config.zero,
match config.end.as_ref() {
Some(config_end) => format!("{}", config_end),
None => "None".into(),
}
))
}
let mut digest_level = 0u32;
let mut digest_step = 1u32;
let mut digest_interval = 0u32;
let mut current = end.clone();
let mut current_begin = begin.clone();
if current_begin != current {
while digest_level != config.config.digest_levels {
// try to use next level digest
let new_digest_level = digest_level + 1;
let new_digest_step = digest_step * config.config.digest_interval;
let new_digest_interval = config.config.digest_interval * {
if digest_interval == 0 {
1
} else {
digest_interval
}
};
let new_digest_begin = config.zero.clone() +
((current.clone() - One::one() - config.zero.clone()) /
new_digest_interval.into()) *
new_digest_interval.into();
let new_digest_end = new_digest_begin.clone() + new_digest_interval.into();
let new_current = new_digest_begin.clone() + new_digest_interval.into();
// check if we met skewed digest
if let Some(skewed_digest_end) = config.end.as_ref() {
if new_digest_end > *skewed_digest_end {
let skewed_digest_start = config.config.prev_max_level_digest_block(
config.zero.clone(),
skewed_digest_end.clone(),
);
if let Some(skewed_digest_start) = skewed_digest_start {
let skewed_digest_range = (skewed_digest_end.clone() -
skewed_digest_start.clone())
.try_into()
.ok()
.expect(
"skewed digest range is always <= max level digest range;\
max level digest range always fits u32; qed",
);
return Ok((
skewed_digest_end.clone(),
skewed_digest_start,
skewed_digest_range,
None,
))
}
}
}
// we can't use next level digest if it touches any unknown (> max) blocks
if new_digest_end > max {
if begin < new_digest_begin {
current_begin = new_digest_begin;
}
break
}
// we can (and will) use this digest
digest_level = new_digest_level;
digest_step = new_digest_step;
digest_interval = new_digest_interval;
current = new_current;
current_begin = new_digest_begin;
// if current digest covers the whole range => no need to use next level digest
if current_begin <= begin && new_digest_end >= end {
break
}
}
}
Ok((current, current_begin, digest_step, Some(digest_level)))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::changes_trie::Configuration;
fn configuration_range<'a>(
config: &'a Configuration,
zero: u64,
) -> ConfigurationRange<'a, u64> {
ConfigurationRange { config, zero, end: None }
}
#[test]
fn lower_bound_max_digest_works() {
let config = Configuration { digest_interval: 4, digest_levels: 2 };
// when config activates at 0
assert_eq!(
lower_bound_max_digest(configuration_range(&config, 0u64), 100_000u64, 20u64, 180u64)
.unwrap(),
(192, 176, 16, Some(2)),
);
// when config activates at 30
assert_eq!(
lower_bound_max_digest(configuration_range(&config, 30u64), 100_000u64, 50u64, 210u64)
.unwrap(),
(222, 206, 16, Some(2)),
);
}
#[test]
fn surface_iterator_works() {
let config = Configuration { digest_interval: 4, digest_levels: 2 };
// when config activates at 0
assert_eq!(
surface_iterator(configuration_range(&config, 0u64), 100_000u64, 40u64, 180u64,)
.unwrap()
.collect::<Vec<_>>(),
vec![
Ok((192, Some(2))),
Ok((176, Some(2))),
Ok((160, Some(2))),
Ok((144, Some(2))),
Ok((128, Some(2))),
Ok((112, Some(2))),
Ok((96, Some(2))),
Ok((80, Some(2))),
Ok((64, Some(2))),
Ok((48, Some(2))),
],
);
// when config activates at 30
assert_eq!(
surface_iterator(configuration_range(&config, 30u64), 100_000u64, 40u64, 180u64,)
.unwrap()
.collect::<Vec<_>>(),
vec![
Ok((190, Some(2))),
Ok((174, Some(2))),
Ok((158, Some(2))),
Ok((142, Some(2))),
Ok((126, Some(2))),
Ok((110, Some(2))),
Ok((94, Some(2))),
Ok((78, Some(2))),
Ok((62, Some(2))),
Ok((46, Some(2))),
],
);
// when config activates at 0 AND max block is before next digest
assert_eq!(
surface_iterator(configuration_range(&config, 0u64), 183u64, 40u64, 183u64)
.unwrap()
.collect::<Vec<_>>(),
vec![
Ok((183, Some(0))),
Ok((182, Some(0))),
Ok((181, Some(0))),
Ok((180, Some(1))),
Ok((176, Some(2))),
Ok((160, Some(2))),
Ok((144, Some(2))),
Ok((128, Some(2))),
Ok((112, Some(2))),
Ok((96, Some(2))),
Ok((80, Some(2))),
Ok((64, Some(2))),
Ok((48, Some(2))),
],
);
}
#[test]
fn surface_iterator_works_with_skewed_digest() {
let config = Configuration { digest_interval: 4, digest_levels: 2 };
let mut config_range = configuration_range(&config, 0u64);
// when config activates at 0 AND ends at 170
config_range.end = Some(170);
assert_eq!(
surface_iterator(config_range, 100_000u64, 40u64, 170u64)
.unwrap()
.collect::<Vec<_>>(),
vec![
Ok((170, None)),
Ok((160, Some(2))),
Ok((144, Some(2))),
Ok((128, Some(2))),
Ok((112, Some(2))),
Ok((96, Some(2))),
Ok((80, Some(2))),
Ok((64, Some(2))),
Ok((48, Some(2))),
],
);
}
}
substrate/primitives/state-machine/src/ext.rs
+24 -171
View File
@@ -28,8 +28,6 @@ use sp_core::storage::{well_known_keys::is_child_storage_key, ChildInfo, Tracked
use sp_externalities::{Extension, ExtensionStore, Externalities};
use sp_trie::{empty_child_trie_root, trie_types::Layout};
#[cfg(feature = "std")]
use crate::changes_trie::State as ChangesTrieState;
use crate::{log_error, trace, warn, StorageTransactionCache};
use sp_std::{
any::{Any, TypeId},
@@ -90,62 +88,52 @@ impl<B: error::Error, E: error::Error> error::Error for Error<B, E> {
}
/// Wraps a read-only backend, call executor, and current overlayed changes.
pub struct Ext<'a, H, N, B>
pub struct Ext<'a, H, B>
where
H: Hasher,
B: 'a + Backend<H>,
N: crate::changes_trie::BlockNumber,
{
/// The overlayed changes to write to.
overlay: &'a mut OverlayedChanges,
/// The storage backend to read from.
backend: &'a B,
/// The cache for the storage transactions.
storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H, N>,
/// Changes trie state to read from.
#[cfg(feature = "std")]
changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H>,
/// Pseudo-unique id used for tracing.
pub id: u16,
/// Dummy usage of N arg.
_phantom: sp_std::marker::PhantomData<N>,
/// Extensions registered with this instance.
#[cfg(feature = "std")]
extensions: Option<OverlayedExtensions<'a>>,
}
impl<'a, H, N, B> Ext<'a, H, N, B>
impl<'a, H, B> Ext<'a, H, B>
where
H: Hasher,
B: Backend<H>,
N: crate::changes_trie::BlockNumber,
{
/// Create a new `Ext`.
#[cfg(not(feature = "std"))]
pub fn new(
overlay: &'a mut OverlayedChanges,
storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H, N>,
storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H>,
backend: &'a B,
) -> Self {
Ext { overlay, backend, id: 0, storage_transaction_cache, _phantom: Default::default() }
Ext { overlay, backend, id: 0, storage_transaction_cache }
}
/// Create a new `Ext` from overlayed changes and read-only backend
#[cfg(feature = "std")]
pub fn new(
overlay: &'a mut OverlayedChanges,
storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H, N>,
storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H>,
backend: &'a B,
changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
extensions: Option<&'a mut sp_externalities::Extensions>,
) -> Self {
Self {
overlay,
backend,
changes_trie_state,
storage_transaction_cache,
id: rand::random(),
_phantom: Default::default(),
extensions: extensions.map(OverlayedExtensions::new),
}
}
@@ -159,12 +147,11 @@ where
}
#[cfg(test)]
impl<'a, H, N, B> Ext<'a, H, N, B>
impl<'a, H, B> Ext<'a, H, B>
where
H: Hasher,
H::Out: Ord + 'static,
B: 'a + Backend<H>,
N: crate::changes_trie::BlockNumber,
{
pub fn storage_pairs(&self) -> Vec<(StorageKey, StorageValue)> {
use std::collections::HashMap;
@@ -181,12 +168,11 @@ where
}
}
impl<'a, H, N, B> Externalities for Ext<'a, H, N, B>
impl<'a, H, B> Externalities for Ext<'a, H, B>
where
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
B: Backend<H>,
N: crate::changes_trie::BlockNumber,
{
fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>) {
self.overlay.set_offchain_storage(key, value)
@@ -644,54 +630,6 @@ where
.add_transaction_index(IndexOperation::Renew { extrinsic: index, hash: hash.to_vec() });
}
#[cfg(not(feature = "std"))]
fn storage_changes_root(&mut self, _parent_hash: &[u8]) -> Result<Option<Vec<u8>>, ()> {
Ok(None)
}
#[cfg(feature = "std")]
fn storage_changes_root(&mut self, mut parent_hash: &[u8]) -> Result<Option<Vec<u8>>, ()> {
let _guard = guard();
if let Some(ref root) = self.storage_transaction_cache.changes_trie_transaction_storage_root
{
trace!(
target: "state",
method = "ChangesRoot",
ext_id = %HexDisplay::from(&self.id.to_le_bytes()),
parent_hash = %HexDisplay::from(&parent_hash),
?root,
cached = true,
);
Ok(Some(root.encode()))
} else {
let root = self.overlay.changes_trie_root(
self.backend,
self.changes_trie_state.as_ref(),
Decode::decode(&mut parent_hash).map_err(|e| {
trace!(
target: "state",
error = %e,
"Failed to decode changes root parent hash",
)
})?,
true,
self.storage_transaction_cache,
);
trace!(
target: "state",
method = "ChangesRoot",
ext_id = %HexDisplay::from(&self.id.to_le_bytes()),
parent_hash = %HexDisplay::from(&parent_hash),
?root,
cached = false,
);
root.map(|r| r.map(|o| o.encode()))
}
}
fn storage_start_transaction(&mut self) {
self.overlay.start_transaction()
}
@@ -710,13 +648,7 @@ where
self.overlay.rollback_transaction().expect(BENCHMARKING_FN);
}
self.overlay
.drain_storage_changes(
self.backend,
#[cfg(feature = "std")]
None,
Default::default(),
self.storage_transaction_cache,
)
.drain_storage_changes(self.backend, Default::default(), self.storage_transaction_cache)
.expect(EXT_NOT_ALLOWED_TO_FAIL);
self.backend.wipe().expect(EXT_NOT_ALLOWED_TO_FAIL);
self.mark_dirty();
@@ -731,13 +663,7 @@ where
}
let changes = self
.overlay
.drain_storage_changes(
self.backend,
#[cfg(feature = "std")]
None,
Default::default(),
self.storage_transaction_cache,
)
.drain_storage_changes(self.backend, Default::default(), self.storage_transaction_cache)
.expect(EXT_NOT_ALLOWED_TO_FAIL);
self.backend
.commit(
@@ -778,12 +704,11 @@ where
}
}
impl<'a, H, N, B> Ext<'a, H, N, B>
impl<'a, H, B> Ext<'a, H, B>
where
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
B: Backend<H>,
N: crate::changes_trie::BlockNumber,
{
fn limit_remove_from_backend(
&mut self,
@@ -869,12 +794,11 @@ impl<'a> StorageAppend<'a> {
}
#[cfg(not(feature = "std"))]
impl<'a, H, N, B> ExtensionStore for Ext<'a, H, N, B>
impl<'a, H, B> ExtensionStore for Ext<'a, H, B>
where
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
B: Backend<H>,
N: crate::changes_trie::BlockNumber,
{
fn extension_by_type_id(&mut self, _type_id: TypeId) -> Option<&mut dyn Any> {
None
@@ -897,11 +821,10 @@ where
}
#[cfg(feature = "std")]
impl<'a, H, N, B> ExtensionStore for Ext<'a, H, N, B>
impl<'a, H, B> ExtensionStore for Ext<'a, H, B>
where
H: Hasher,
B: 'a + Backend<H>,
N: crate::changes_trie::BlockNumber,
{
fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any> {
self.extensions.as_mut().and_then(|exts| exts.get_mut(type_id))
@@ -938,86 +861,16 @@ where
#[cfg(test)]
mod tests {
use super::*;
use crate::{
changes_trie::{
Configuration as ChangesTrieConfiguration, InMemoryStorage as TestChangesTrieStorage,
},
InMemoryBackend,
};
use crate::InMemoryBackend;
use codec::Encode;
use hex_literal::hex;
use num_traits::Zero;
use sp_core::{
map,
storage::{well_known_keys::EXTRINSIC_INDEX, Storage, StorageChild},
Blake2Hasher, H256,
storage::{Storage, StorageChild},
Blake2Hasher,
};
type TestBackend = InMemoryBackend<Blake2Hasher>;
type TestExt<'a> = Ext<'a, Blake2Hasher, u64, TestBackend>;
fn prepare_overlay_with_changes() -> OverlayedChanges {
let mut changes = OverlayedChanges::default();
changes.set_collect_extrinsics(true);
changes.set_extrinsic_index(1);
changes.set_storage(vec![1], Some(vec![100]));
changes.set_storage(EXTRINSIC_INDEX.to_vec(), Some(3u32.encode()));
changes.set_offchain_storage(b"k1", Some(b"v1"));
changes.set_offchain_storage(b"k2", Some(b"v2"));
changes
}
fn changes_trie_config() -> ChangesTrieConfiguration {
ChangesTrieConfiguration { digest_interval: 0, digest_levels: 0 }
}
#[test]
fn storage_changes_root_is_none_when_storage_is_not_provided() {
let mut overlay = prepare_overlay_with_changes();
let mut cache = StorageTransactionCache::default();
let backend = TestBackend::default();
let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None);
}
#[test]
fn storage_changes_root_is_none_when_state_is_not_provided() {
let mut overlay = prepare_overlay_with_changes();
let mut cache = StorageTransactionCache::default();
let backend = TestBackend::default();
let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None);
}
#[test]
fn storage_changes_root_is_some_when_extrinsic_changes_are_non_empty() {
let mut overlay = prepare_overlay_with_changes();
let mut cache = StorageTransactionCache::default();
let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage));
let backend = TestBackend::default();
let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None);
assert_eq!(
ext.storage_changes_root(&H256::default().encode()).unwrap(),
Some(hex!("bb0c2ef6e1d36d5490f9766cfcc7dfe2a6ca804504c3bb206053890d6dd02376").to_vec()),
);
}
#[test]
fn storage_changes_root_is_some_when_extrinsic_changes_are_empty() {
let mut overlay = prepare_overlay_with_changes();
let mut cache = StorageTransactionCache::default();
overlay.set_collect_extrinsics(false);
overlay.set_storage(vec![1], None);
let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage));
let backend = TestBackend::default();
let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None);
assert_eq!(
ext.storage_changes_root(&H256::default().encode()).unwrap(),
Some(hex!("96f5aae4690e7302737b6f9b7f8567d5bbb9eac1c315f80101235a92d9ec27f4").to_vec()),
);
}
type TestExt<'a> = Ext<'a, Blake2Hasher, TestBackend>;
#[test]
fn next_storage_key_works() {
@@ -1035,7 +888,7 @@ mod tests {
}
.into();
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None);
// next_backend < next_overlay
assert_eq!(ext.next_storage_key(&[5]), Some(vec![10]));
@@ -1051,7 +904,7 @@ mod tests {
drop(ext);
overlay.set_storage(vec![50], Some(vec![50]));
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None);
// next_overlay exist but next_backend doesn't exist
assert_eq!(ext.next_storage_key(&[40]), Some(vec![50]));
@@ -1079,7 +932,7 @@ mod tests {
}
.into();
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None);
assert_eq!(ext.next_storage_key(&[5]), Some(vec![30]));
@@ -1110,7 +963,7 @@ mod tests {
}
.into();
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None);
// next_backend < next_overlay
assert_eq!(ext.next_child_storage_key(child_info, &[5]), Some(vec![10]));
@@ -1126,7 +979,7 @@ mod tests {
drop(ext);
overlay.set_child_storage(child_info, vec![50], Some(vec![50]));
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None);
// next_overlay exist but next_backend doesn't exist
assert_eq!(ext.next_child_storage_key(child_info, &[40]), Some(vec![50]));
@@ -1155,7 +1008,7 @@ mod tests {
}
.into();
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None);
assert_eq!(ext.child_storage(child_info, &[10]), Some(vec![10]));
assert_eq!(
@@ -1192,7 +1045,7 @@ mod tests {
}
.into();
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None);
use sp_core::storage::well_known_keys;
let mut ext = ext;
substrate/primitives/state-machine/src/lib.rs
+32 -178
View File
@@ -23,8 +23,6 @@
pub mod backend;
#[cfg(feature = "std")]
mod basic;
#[cfg(feature = "std")]
mod changes_trie;
mod error;
mod ext;
#[cfg(feature = "std")]
@@ -140,28 +138,10 @@ pub use crate::{
};
pub use error::{Error, ExecutionError};
#[cfg(not(feature = "std"))]
mod changes_trie {
/// Stub for change trie block number until
/// change trie move to no_std.
pub trait BlockNumber {}
impl<N> BlockNumber for N {}
}
#[cfg(feature = "std")]
mod std_reexport {
pub use crate::{
basic::BasicExternalities,
changes_trie::{
disabled_state as disabled_changes_trie_state, key_changes, key_changes_proof,
key_changes_proof_check, key_changes_proof_check_with_db, prune as prune_changes_tries,
AnchorBlockId as ChangesTrieAnchorBlockId, BlockNumber as ChangesTrieBlockNumber,
BuildCache as ChangesTrieBuildCache, CacheAction as ChangesTrieCacheAction,
ConfigurationRange as ChangesTrieConfigurationRange,
InMemoryStorage as InMemoryChangesTrieStorage, RootsStorage as ChangesTrieRootsStorage,
State as ChangesTrieState, Storage as ChangesTrieStorage,
},
error::{Error, ExecutionError},
in_memory_backend::new_in_mem,
proving_backend::{
@@ -205,10 +185,6 @@ mod execution {
/// Default handler of the execution manager.
pub type DefaultHandler<R, E> = fn(CallResult<R, E>, CallResult<R, E>) -> CallResult<R, E>;
/// Type of changes trie transaction.
pub type ChangesTrieTransaction<H, N> =
(MemoryDB<H>, ChangesTrieCacheAction<<H as Hasher>::Out, N>);
/// Trie backend with in-memory storage.
pub type InMemoryBackend<H> = TrieBackend<MemoryDB<H>, H>;
@@ -308,11 +284,10 @@ mod execution {
}
/// The substrate state machine.
pub struct StateMachine<'a, B, H, N, Exec>
pub struct StateMachine<'a, B, H, Exec>
where
H: Hasher,
B: Backend<H>,
N: ChangesTrieBlockNumber,
{
backend: &'a B,
exec: &'a Exec,
@@ -320,8 +295,7 @@ mod execution {
call_data: &'a [u8],
overlay: &'a mut OverlayedChanges,
extensions: Extensions,
changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
storage_transaction_cache: Option<&'a mut StorageTransactionCache<B::Transaction, H, N>>,
storage_transaction_cache: Option<&'a mut StorageTransactionCache<B::Transaction, H>>,
runtime_code: &'a RuntimeCode<'a>,
stats: StateMachineStats,
/// The hash of the block the state machine will be executed on.
@@ -330,29 +304,26 @@ mod execution {
parent_hash: Option<H::Out>,
}
impl<'a, B, H, N, Exec> Drop for StateMachine<'a, B, H, N, Exec>
impl<'a, B, H, Exec> Drop for StateMachine<'a, B, H, Exec>
where
H: Hasher,
B: Backend<H>,
N: ChangesTrieBlockNumber,
{
fn drop(&mut self) {
self.backend.register_overlay_stats(&self.stats);
}
}
impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec>
impl<'a, B, H, Exec> StateMachine<'a, B, H, Exec>
where
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
Exec: CodeExecutor + Clone + 'static,
B: Backend<H>,
N: crate::changes_trie::BlockNumber,
{
/// Creates new substrate state machine.
pub fn new(
backend: &'a B,
changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
overlay: &'a mut OverlayedChanges,
exec: &'a Exec,
method: &'a str,
@@ -371,7 +342,6 @@ mod execution {
call_data,
extensions,
overlay,
changes_trie_state,
storage_transaction_cache: None,
runtime_code,
stats: StateMachineStats::default(),
@@ -386,7 +356,7 @@ mod execution {
/// build that will be cached.
pub fn with_storage_transaction_cache(
mut self,
cache: Option<&'a mut StorageTransactionCache<B::Transaction, H, N>>,
cache: Option<&'a mut StorageTransactionCache<B::Transaction, H>>,
) -> Self {
self.storage_transaction_cache = cache;
self
@@ -439,13 +409,7 @@ mod execution {
.enter_runtime()
.expect("StateMachine is never called from the runtime; qed");
let mut ext = Ext::new(
self.overlay,
cache,
self.backend,
self.changes_trie_state.clone(),
Some(&mut self.extensions),
);
let mut ext = Ext::new(self.overlay, cache, self.backend, Some(&mut self.extensions));
let ext_id = ext.id;
@@ -562,9 +526,6 @@ mod execution {
CallResult<R, Exec::Error>,
) -> CallResult<R, Exec::Error>,
{
let changes_tries_enabled = self.changes_trie_state.is_some();
self.overlay.set_collect_extrinsics(changes_tries_enabled);
let result = {
match manager {
ExecutionManager::Both(on_consensus_failure) => self
@@ -588,7 +549,7 @@ mod execution {
}
/// Prove execution using the given state backend, overlayed changes, and call executor.
pub fn prove_execution<B, H, N, Exec, Spawn>(
pub fn prove_execution<B, H, Exec, Spawn>(
backend: &mut B,
overlay: &mut OverlayedChanges,
exec: &Exec,
@@ -602,13 +563,12 @@ mod execution {
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
Exec: CodeExecutor + Clone + 'static,
N: crate::changes_trie::BlockNumber,
Spawn: SpawnNamed + Send + 'static,
{
let trie_backend = backend
.as_trie_backend()
.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<dyn Error>)?;
prove_execution_on_trie_backend::<_, _, N, _, _>(
prove_execution_on_trie_backend::<_, _, _, _>(
trie_backend,
overlay,
exec,
@@ -628,7 +588,7 @@ mod execution {
///
/// Note: changes to code will be in place if this call is made again. For running partial
/// blocks (e.g. a transaction at a time), ensure a different method is used.
pub fn prove_execution_on_trie_backend<S, H, N, Exec, Spawn>(
pub fn prove_execution_on_trie_backend<S, H, Exec, Spawn>(
trie_backend: &TrieBackend<S, H>,
overlay: &mut OverlayedChanges,
exec: &Exec,
@@ -642,13 +602,11 @@ mod execution {
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
Exec: CodeExecutor + 'static + Clone,
N: crate::changes_trie::BlockNumber,
Spawn: SpawnNamed + Send + 'static,
{
let proving_backend = proving_backend::ProvingBackend::new(trie_backend);
let mut sm = StateMachine::<_, H, N, Exec>::new(
let mut sm = StateMachine::<_, H, Exec>::new(
&proving_backend,
None,
overlay,
exec,
method,
@@ -667,7 +625,7 @@ mod execution {
}
/// Check execution proof, generated by `prove_execution` call.
pub fn execution_proof_check<H, N, Exec, Spawn>(
pub fn execution_proof_check<H, Exec, Spawn>(
root: H::Out,
proof: StorageProof,
overlay: &mut OverlayedChanges,
@@ -681,11 +639,10 @@ mod execution {
H: Hasher,
Exec: CodeExecutor + Clone + 'static,
H::Out: Ord + 'static + codec::Codec,
N: crate::changes_trie::BlockNumber,
Spawn: SpawnNamed + Send + 'static,
{
let trie_backend = create_proof_check_backend::<H>(root.into(), proof)?;
execution_proof_check_on_trie_backend::<_, N, _, _>(
execution_proof_check_on_trie_backend::<_, _, _>(
&trie_backend,
overlay,
exec,
@@ -697,7 +654,7 @@ mod execution {
}
/// Check execution proof on proving backend, generated by `prove_execution` call.
pub fn execution_proof_check_on_trie_backend<H, N, Exec, Spawn>(
pub fn execution_proof_check_on_trie_backend<H, Exec, Spawn>(
trie_backend: &TrieBackend<MemoryDB<H>, H>,
overlay: &mut OverlayedChanges,
exec: &Exec,
@@ -710,12 +667,10 @@ mod execution {
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
Exec: CodeExecutor + Clone + 'static,
N: crate::changes_trie::BlockNumber,
Spawn: SpawnNamed + Send + 'static,
{
let mut sm = StateMachine::<_, H, N, Exec>::new(
let mut sm = StateMachine::<_, H, Exec>::new(
trie_backend,
None,
overlay,
exec,
method,
@@ -1390,7 +1345,7 @@ mod execution {
#[cfg(test)]
mod tests {
use super::{changes_trie::Configuration as ChangesTrieConfig, ext::Ext, *};
use super::{ext::Ext, *};
use crate::execution::CallResult;
use codec::{Decode, Encode};
use sp_core::{
@@ -1409,7 +1364,6 @@ mod tests {
#[derive(Clone)]
struct DummyCodeExecutor {
change_changes_trie_config: bool,
native_available: bool,
native_succeeds: bool,
fallback_succeeds: bool,
@@ -1430,13 +1384,6 @@ mod tests {
use_native: bool,
native_call: Option<NC>,
) -> (CallResult<R, Self::Error>, bool) {
if self.change_changes_trie_config {
ext.place_storage(
sp_core::storage::well_known_keys::CHANGES_TRIE_CONFIG.to_vec(),
Some(ChangesTrieConfig { digest_interval: 777, digest_levels: 333 }.encode()),
);
}
let using_native = use_native && self.native_available;
match (using_native, self.native_succeeds, self.fallback_succeeds, native_call) {
(true, true, _, Some(call)) => {
@@ -1472,10 +1419,8 @@ mod tests {
let mut state_machine = StateMachine::new(
&backend,
changes_trie::disabled_state::<_, u64>(),
&mut overlayed_changes,
&DummyCodeExecutor {
change_changes_trie_config: false,
native_available: true,
native_succeeds: true,
fallback_succeeds: true,
@@ -1498,10 +1443,8 @@ mod tests {
let mut state_machine = StateMachine::new(
&backend,
changes_trie::disabled_state::<_, u64>(),
&mut overlayed_changes,
&DummyCodeExecutor {
change_changes_trie_config: false,
native_available: true,
native_succeeds: true,
fallback_succeeds: true,
@@ -1525,10 +1468,8 @@ mod tests {
let mut state_machine = StateMachine::new(
&backend,
changes_trie::disabled_state::<_, u64>(),
&mut overlayed_changes,
&DummyCodeExecutor {
change_changes_trie_config: false,
native_available: true,
native_succeeds: true,
fallback_succeeds: false,
@@ -1555,7 +1496,6 @@ mod tests {
#[test]
fn prove_execution_and_proof_check_works() {
let executor = DummyCodeExecutor {
change_changes_trie_config: false,
native_available: true,
native_succeeds: true,
fallback_succeeds: true,
@@ -1564,7 +1504,7 @@ mod tests {
// fetch execution proof from 'remote' full node
let mut remote_backend = trie_backend::tests::test_trie();
let remote_root = remote_backend.storage_root(std::iter::empty()).0;
let (remote_result, remote_proof) = prove_execution::<_, _, u64, _, _>(
let (remote_result, remote_proof) = prove_execution(
&mut remote_backend,
&mut Default::default(),
&executor,
@@ -1576,7 +1516,7 @@ mod tests {
.unwrap();
// check proof locally
let local_result = execution_proof_check::<BlakeTwo256, u64, _, _>(
let local_result = execution_proof_check::<BlakeTwo256, _, _>(
remote_root,
remote_proof,
&mut Default::default(),
@@ -1614,13 +1554,7 @@ mod tests {
let overlay_limit = overlay.clone();
{
let mut cache = StorageTransactionCache::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
backend,
changes_trie::disabled_state::<_, u64>(),
None,
);
let mut ext = Ext::new(&mut overlay, &mut cache, backend, None);
ext.clear_prefix(b"ab", None);
}
overlay.commit_transaction().unwrap();
@@ -1644,13 +1578,7 @@ mod tests {
let mut overlay = overlay_limit;
{
let mut cache = StorageTransactionCache::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
backend,
changes_trie::disabled_state::<_, u64>(),
None,
);
let mut ext = Ext::new(&mut overlay, &mut cache, backend, None);
assert_eq!((false, 1), ext.clear_prefix(b"ab", Some(1)));
}
overlay.commit_transaction().unwrap();
@@ -1692,13 +1620,7 @@ mod tests {
{
let mut cache = StorageTransactionCache::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
&backend,
changes_trie::disabled_state::<_, u64>(),
None,
);
let mut ext = Ext::new(&mut overlay, &mut cache, &backend, None);
assert_eq!(ext.kill_child_storage(&child_info, Some(2)), (false, 2));
}
@@ -1733,13 +1655,7 @@ mod tests {
let backend = InMemoryBackend::<BlakeTwo256>::from(initial);
let mut overlay = OverlayedChanges::default();
let mut cache = StorageTransactionCache::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
&backend,
changes_trie::disabled_state::<_, u64>(),
None,
);
let mut ext = Ext::new(&mut overlay, &mut cache, &backend, None);
assert_eq!(ext.kill_child_storage(&child_info, Some(0)), (false, 0));
assert_eq!(ext.kill_child_storage(&child_info, Some(1)), (false, 1));
assert_eq!(ext.kill_child_storage(&child_info, Some(2)), (false, 2));
@@ -1758,13 +1674,7 @@ mod tests {
let backend = state.as_trie_backend().unwrap();
let mut overlay = OverlayedChanges::default();
let mut cache = StorageTransactionCache::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
backend,
changes_trie::disabled_state::<_, u64>(),
None,
);
let mut ext = Ext::new(&mut overlay, &mut cache, backend, None);
ext.set_child_storage(child_info, b"abc".to_vec(), b"def".to_vec());
assert_eq!(ext.child_storage(child_info, b"abc"), Some(b"def".to_vec()));
@@ -1781,26 +1691,14 @@ mod tests {
let mut overlay = OverlayedChanges::default();
let mut cache = StorageTransactionCache::default();
{
let mut ext = Ext::new(
&mut overlay,
&mut cache,
backend,
changes_trie::disabled_state::<_, u64>(),
None,
);
let mut ext = Ext::new(&mut overlay, &mut cache, backend, None);
ext.storage_append(key.clone(), reference_data[0].encode());
assert_eq!(ext.storage(key.as_slice()), Some(vec![reference_data[0].clone()].encode()));
}
overlay.start_transaction();
{
let mut ext = Ext::new(
&mut overlay,
&mut cache,
backend,
changes_trie::disabled_state::<_, u64>(),
None,
);
let mut ext = Ext::new(&mut overlay, &mut cache, backend, None);
for i in reference_data.iter().skip(1) {
ext.storage_append(key.clone(), i.encode());
@@ -1809,13 +1707,7 @@ mod tests {
}
overlay.rollback_transaction().unwrap();
{
let ext = Ext::new(
&mut overlay,
&mut cache,
backend,
changes_trie::disabled_state::<_, u64>(),
None,
);
let ext = Ext::new(&mut overlay, &mut cache, backend, None);
assert_eq!(ext.storage(key.as_slice()), Some(vec![reference_data[0].clone()].encode()));
}
}
@@ -1837,13 +1729,7 @@ mod tests {
// For example, block initialization with event.
{
let mut ext = Ext::new(
&mut overlay,
&mut cache,
backend,
changes_trie::disabled_state::<_, u64>(),
None,
);
let mut ext = Ext::new(&mut overlay, &mut cache, backend, None);
ext.clear_storage(key.as_slice());
ext.storage_append(key.clone(), Item::InitializationItem.encode());
}
@@ -1851,13 +1737,7 @@ mod tests {
// For example, first transaction resulted in panic during block building
{
let mut ext = Ext::new(
&mut overlay,
&mut cache,
backend,
changes_trie::disabled_state::<_, u64>(),
None,
);
let mut ext = Ext::new(&mut overlay, &mut cache, backend, None);
assert_eq!(ext.storage(key.as_slice()), Some(vec![Item::InitializationItem].encode()));
@@ -1872,13 +1752,7 @@ mod tests {
// Then we apply next transaction which is valid this time.
{
let mut ext = Ext::new(
&mut overlay,
&mut cache,
backend,
changes_trie::disabled_state::<_, u64>(),
None,
);
let mut ext = Ext::new(&mut overlay, &mut cache, backend, None);
assert_eq!(ext.storage(key.as_slice()), Some(vec![Item::InitializationItem].encode()));
@@ -1893,13 +1767,7 @@ mod tests {
// Then only initlaization item and second (committed) item should persist.
{
let ext = Ext::new(
&mut overlay,
&mut cache,
backend,
changes_trie::disabled_state::<_, u64>(),
None,
);
let ext = Ext::new(&mut overlay, &mut cache, backend, None);
assert_eq!(
ext.storage(key.as_slice()),
Some(vec![Item::InitializationItem, Item::CommitedItem].encode()),
@@ -2214,13 +2082,7 @@ mod tests {
let mut transaction = {
let backend = test_trie();
let mut cache = StorageTransactionCache::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
&backend,
changes_trie::disabled_state::<_, u64>(),
None,
);
let mut ext = Ext::new(&mut overlay, &mut cache, &backend, None);
ext.set_child_storage(&child_info_1, b"abc".to_vec(), b"def".to_vec());
ext.set_child_storage(&child_info_2, b"abc".to_vec(), b"def".to_vec());
ext.storage_root();
@@ -2257,13 +2119,7 @@ mod tests {
{
let mut cache = StorageTransactionCache::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
backend,
changes_trie::disabled_state::<_, u64>(),
None,
);
let mut ext = Ext::new(&mut overlay, &mut cache, backend, None);
assert_eq!(ext.storage(b"bbb"), Some(vec![]));
assert_eq!(ext.storage(b"ccc"), Some(vec![]));
ext.clear_storage(b"ccc");
@@ -2286,10 +2142,8 @@ mod tests {
let mut state_machine = StateMachine::new(
&backend,
changes_trie::disabled_state::<_, u64>(),
&mut overlayed_changes,
&DummyCodeExecutor {
change_changes_trie_config: false,
native_available: true,
native_succeeds: true,
fallback_succeeds: false,
@@ -2301,7 +2155,7 @@ mod tests {
TaskExecutor::new(),
);
let run_state_machine = |state_machine: &mut StateMachine<_, _, _, _>| {
let run_state_machine = |state_machine: &mut StateMachine<_, _, _>| {
state_machine
.execute_using_consensus_failure_handler::<fn(_, _) -> _, _, _>(
ExecutionManager::NativeWhenPossible,
substrate/primitives/state-machine/src/overlayed_changes/changeset.rs
-1
View File
@@ -69,7 +69,6 @@ struct InnerValue<V> {
/// Current value. None if value has been deleted.
value: V,
/// The set of extrinsic indices where the values has been changed.
/// Is filled only if runtime has announced changes trie support.
extrinsics: Extrinsics,
}
substrate/primitives/state-machine/src/overlayed_changes/mod.rs
+21 -125
View File
@@ -21,12 +21,7 @@ mod changeset;
mod offchain;
use self::changeset::OverlayedChangeSet;
use crate::{backend::Backend, changes_trie::BlockNumber, stats::StateMachineStats, DefaultError};
#[cfg(feature = "std")]
use crate::{
changes_trie::{build_changes_trie, State as ChangesTrieState},
ChangesTrieTransaction,
};
use crate::{backend::Backend, stats::StateMachineStats, DefaultError};
use codec::{Decode, Encode};
use hash_db::Hasher;
pub use offchain::OffchainOverlayedChanges;
@@ -134,7 +129,7 @@ pub enum IndexOperation {
///
/// This contains all the changes to the storage and transactions to apply theses changes to the
/// backend.
pub struct StorageChanges<Transaction, H: Hasher, N: BlockNumber> {
pub struct StorageChanges<Transaction, H: Hasher> {
/// All changes to the main storage.
///
/// A value of `None` means that it was deleted.
@@ -150,22 +145,13 @@ pub struct StorageChanges<Transaction, H: Hasher, N: BlockNumber> {
pub transaction: Transaction,
/// The storage root after applying the transaction.
pub transaction_storage_root: H::Out,
/// Contains the transaction for the backend for the changes trie.
///
/// If changes trie is disabled the value is set to `None`.
#[cfg(feature = "std")]
pub changes_trie_transaction: Option<ChangesTrieTransaction<H, N>>,
/// Phantom data for block number until change trie support no_std.
#[cfg(not(feature = "std"))]
pub _ph: sp_std::marker::PhantomData<N>,
/// Changes to the transaction index,
#[cfg(feature = "std")]
pub transaction_index_changes: Vec<IndexOperation>,
}
#[cfg(feature = "std")]
impl<Transaction, H: Hasher, N: BlockNumber> StorageChanges<Transaction, H, N> {
impl<Transaction, H: Hasher> StorageChanges<Transaction, H> {
/// Deconstruct into the inner values
pub fn into_inner(
self,
@@ -175,7 +161,6 @@ impl<Transaction, H: Hasher, N: BlockNumber> StorageChanges<Transaction, H, N> {
OffchainChangesCollection,
Transaction,
H::Out,
Option<ChangesTrieTransaction<H, N>>,
Vec<IndexOperation>,
) {
(
@@ -184,58 +169,35 @@ impl<Transaction, H: Hasher, N: BlockNumber> StorageChanges<Transaction, H, N> {
self.offchain_storage_changes,
self.transaction,
self.transaction_storage_root,
self.changes_trie_transaction,
self.transaction_index_changes,
)
}
}
/// The storage transaction are calculated as part of the `storage_root` and
/// `changes_trie_storage_root`. These transactions can be reused for importing the block into the
/// Storage transactions are calculated as part of the `storage_root`.
/// These transactions can be reused for importing the block into the
/// storage. So, we cache them to not require a recomputation of those transactions.
pub struct StorageTransactionCache<Transaction, H: Hasher, N: BlockNumber> {
pub struct StorageTransactionCache<Transaction, H: Hasher> {
/// Contains the changes for the main and the child storages as one transaction.
pub(crate) transaction: Option<Transaction>,
/// The storage root after applying the transaction.
pub(crate) transaction_storage_root: Option<H::Out>,
/// Contains the changes trie transaction.
#[cfg(feature = "std")]
pub(crate) changes_trie_transaction: Option<Option<ChangesTrieTransaction<H, N>>>,
/// The storage root after applying the changes trie transaction.
#[cfg(feature = "std")]
pub(crate) changes_trie_transaction_storage_root: Option<Option<H::Out>>,
/// Phantom data for block number until change trie support no_std.
#[cfg(not(feature = "std"))]
pub(crate) _ph: sp_std::marker::PhantomData<N>,
}
impl<Transaction, H: Hasher, N: BlockNumber> StorageTransactionCache<Transaction, H, N> {
impl<Transaction, H: Hasher> StorageTransactionCache<Transaction, H> {
/// Reset the cached transactions.
pub fn reset(&mut self) {
*self = Self::default();
}
}
impl<Transaction, H: Hasher, N: BlockNumber> Default
for StorageTransactionCache<Transaction, H, N>
{
impl<Transaction, H: Hasher> Default for StorageTransactionCache<Transaction, H> {
fn default() -> Self {
Self {
transaction: None,
transaction_storage_root: None,
#[cfg(feature = "std")]
changes_trie_transaction: None,
#[cfg(feature = "std")]
changes_trie_transaction_storage_root: None,
#[cfg(not(feature = "std"))]
_ph: Default::default(),
}
Self { transaction: None, transaction_storage_root: None }
}
}
impl<Transaction: Default, H: Hasher, N: BlockNumber> Default
for StorageChanges<Transaction, H, N>
{
impl<Transaction: Default, H: Hasher> Default for StorageChanges<Transaction, H> {
fn default() -> Self {
Self {
main_storage_changes: Default::default(),
@@ -244,10 +206,6 @@ impl<Transaction: Default, H: Hasher, N: BlockNumber> Default
transaction: Default::default(),
transaction_storage_root: Default::default(),
#[cfg(feature = "std")]
changes_trie_transaction: None,
#[cfg(not(feature = "std"))]
_ph: Default::default(),
#[cfg(feature = "std")]
transaction_index_changes: Default::default(),
}
}
@@ -539,27 +497,25 @@ impl OverlayedChanges {
/// Convert this instance with all changes into a [`StorageChanges`] instance.
#[cfg(feature = "std")]
pub fn into_storage_changes<B: Backend<H>, H: Hasher, N: BlockNumber>(
pub fn into_storage_changes<B: Backend<H>, H: Hasher>(
mut self,
backend: &B,
changes_trie_state: Option<&ChangesTrieState<H, N>>,
parent_hash: H::Out,
mut cache: StorageTransactionCache<B::Transaction, H, N>,
) -> Result<StorageChanges<B::Transaction, H, N>, DefaultError>
mut cache: StorageTransactionCache<B::Transaction, H>,
) -> Result<StorageChanges<B::Transaction, H>, DefaultError>
where
H::Out: Ord + Encode + 'static,
{
self.drain_storage_changes(backend, changes_trie_state, parent_hash, &mut cache)
self.drain_storage_changes(backend, parent_hash, &mut cache)
}
/// Drain all changes into a [`StorageChanges`] instance. Leave empty overlay in place.
pub fn drain_storage_changes<B: Backend<H>, H: Hasher, N: BlockNumber>(
pub fn drain_storage_changes<B: Backend<H>, H: Hasher>(
&mut self,
backend: &B,
#[cfg(feature = "std")] changes_trie_state: Option<&ChangesTrieState<H, N>>,
parent_hash: H::Out,
mut cache: &mut StorageTransactionCache<B::Transaction, H, N>,
) -> Result<StorageChanges<B::Transaction, H, N>, DefaultError>
_parent_hash: H::Out,
mut cache: &mut StorageTransactionCache<B::Transaction, H>,
) -> Result<StorageChanges<B::Transaction, H>, DefaultError>
where
H::Out: Ord + Encode + 'static,
{
@@ -574,21 +530,6 @@ impl OverlayedChanges {
.and_then(|t| cache.transaction_storage_root.take().map(|tr| (t, tr)))
.expect("Transaction was be generated as part of `storage_root`; qed");
// If the transaction does not exist, we generate it.
#[cfg(feature = "std")]
if cache.changes_trie_transaction.is_none() {
self.changes_trie_root(backend, changes_trie_state, parent_hash, false, &mut cache)
.map_err(|_| "Failed to generate changes trie transaction")?;
}
#[cfg(not(feature = "std"))]
let _ = parent_hash;
#[cfg(feature = "std")]
let changes_trie_transaction = cache
.changes_trie_transaction
.take()
.expect("Changes trie transaction was generated by `changes_trie_root`; qed");
let (main_storage_changes, child_storage_changes) = self.drain_committed();
let offchain_storage_changes = self.offchain_drain_committed().collect();
@@ -604,11 +545,7 @@ impl OverlayedChanges {
transaction,
transaction_storage_root,
#[cfg(feature = "std")]
changes_trie_transaction,
#[cfg(feature = "std")]
transaction_index_changes,
#[cfg(not(feature = "std"))]
_ph: Default::default(),
})
}
@@ -639,10 +576,10 @@ impl OverlayedChanges {
/// as seen by the current transaction.
///
/// Returns the storage root and caches storage transaction in the given `cache`.
pub fn storage_root<H: Hasher, N: BlockNumber, B: Backend<H>>(
pub fn storage_root<H: Hasher, B: Backend<H>>(
&self,
backend: &B,
cache: &mut StorageTransactionCache<B::Transaction, H, N>,
cache: &mut StorageTransactionCache<B::Transaction, H>,
) -> H::Out
where
H::Out: Ord + Encode,
@@ -660,40 +597,6 @@ impl OverlayedChanges {
root
}
/// Generate the changes trie root.
///
/// Returns the changes trie root and caches the storage transaction into the given `cache`.
///
/// # Panics
///
/// Panics on storage error, when `panic_on_storage_error` is set.
#[cfg(feature = "std")]
pub fn changes_trie_root<'a, H: Hasher, N: BlockNumber, B: Backend<H>>(
&self,
backend: &B,
changes_trie_state: Option<&'a ChangesTrieState<'a, H, N>>,
parent_hash: H::Out,
panic_on_storage_error: bool,
cache: &mut StorageTransactionCache<B::Transaction, H, N>,
) -> Result<Option<H::Out>, ()>
where
H::Out: Ord + Encode + 'static,
{
build_changes_trie::<_, H, N>(
backend,
changes_trie_state,
self,
parent_hash,
panic_on_storage_error,
)
.map(|r| {
let root = r.as_ref().map(|r| r.1).clone();
cache.changes_trie_transaction = Some(r.map(|(db, _, cache)| (db, cache)));
cache.changes_trie_transaction_storage_root = Some(root);
root
})
}
/// Returns an iterator over the keys (in lexicographic order) following `key` (excluding `key`)
/// alongside its value.
pub fn iter_after(&self, key: &[u8]) -> impl Iterator<Item = (&[u8], &OverlayedValue)> {
@@ -937,7 +840,6 @@ mod tests {
.collect();
let backend = InMemoryBackend::<Blake2Hasher>::from(initial);
let mut overlay = OverlayedChanges::default();
overlay.set_collect_extrinsics(false);
overlay.start_transaction();
overlay.set_storage(b"dog".to_vec(), Some(b"puppy".to_vec()));
@@ -950,13 +852,7 @@ mod tests {
overlay.set_storage(b"doug".to_vec(), None);
let mut cache = StorageTransactionCache::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
&backend,
crate::changes_trie::disabled_state::<_, u64>(),
None,
);
let mut ext = Ext::new(&mut overlay, &mut cache, &backend, None);
const ROOT: [u8; 32] =
hex!("39245109cef3758c2eed2ccba8d9b370a917850af3824bc8348d505df2c298fa");
substrate/primitives/state-machine/src/read_only.rs
-4
View File
@@ -153,10 +153,6 @@ impl<'a, H: Hasher, B: 'a + Backend<H>> Externalities for ReadOnlyExternalities<
unimplemented!("child_storage_root is not supported in ReadOnlyExternalities")
}
fn storage_changes_root(&mut self, _parent: &[u8]) -> Result<Option<Vec<u8>>, ()> {
unimplemented!("storage_changes_root is not supported in ReadOnlyExternalities")
}
fn storage_start_transaction(&mut self) {
unimplemented!("Transactions are not supported by ReadOnlyExternalities");
}
substrate/primitives/state-machine/src/testing.rs
+21 -52
View File
@@ -23,21 +23,15 @@ use std::{
};
use crate::{
backend::Backend,
changes_trie::{
BlockNumber as ChangesTrieBlockNumber, Configuration as ChangesTrieConfiguration,
InMemoryStorage as ChangesTrieInMemoryStorage, State as ChangesTrieState,
},
ext::Ext,
InMemoryBackend, OverlayedChanges, StorageKey, StorageTransactionCache, StorageValue,
backend::Backend, ext::Ext, InMemoryBackend, OverlayedChanges, StorageKey,
StorageTransactionCache, StorageValue,
};
use codec::Decode;
use hash_db::Hasher;
use sp_core::{
offchain::testing::TestPersistentOffchainDB,
storage::{
well_known_keys::{is_child_storage_key, CHANGES_TRIE_CONFIG, CODE},
well_known_keys::{is_child_storage_key, CODE},
Storage,
},
testing::TaskExecutor,
@@ -46,7 +40,7 @@ use sp_core::{
use sp_externalities::{Extension, ExtensionStore, Extensions};
/// Simple HashMap-based Externalities impl.
pub struct TestExternalities<H: Hasher, N: ChangesTrieBlockNumber = u64>
pub struct TestExternalities<H: Hasher>
where
H::Out: codec::Codec + Ord,
{
@@ -54,33 +48,23 @@ where
overlay: OverlayedChanges,
offchain_db: TestPersistentOffchainDB,
storage_transaction_cache:
StorageTransactionCache<<InMemoryBackend<H> as Backend<H>>::Transaction, H, N>,
StorageTransactionCache<<InMemoryBackend<H> as Backend<H>>::Transaction, H>,
/// Storage backend.
pub backend: InMemoryBackend<H>,
changes_trie_config: Option<ChangesTrieConfiguration>,
changes_trie_storage: ChangesTrieInMemoryStorage<H, N>,
/// Extensions.
pub extensions: Extensions,
}
impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
impl<H: Hasher> TestExternalities<H>
where
H::Out: Ord + 'static + codec::Codec,
{
/// Get externalities implementation.
pub fn ext(&mut self) -> Ext<H, N, InMemoryBackend<H>> {
pub fn ext(&mut self) -> Ext<H, InMemoryBackend<H>> {
Ext::new(
&mut self.overlay,
&mut self.storage_transaction_cache,
&self.backend,
match self.changes_trie_config.clone() {
Some(config) => Some(ChangesTrieState {
config,
zero: 0.into(),
storage: &self.changes_trie_storage,
}),
None => None,
},
Some(&mut self.extensions),
)
}
@@ -97,12 +81,7 @@ where
/// Create a new instance of `TestExternalities` with code and storage.
pub fn new_with_code(code: &[u8], mut storage: Storage) -> Self {
let mut overlay = OverlayedChanges::default();
let changes_trie_config = storage
.top
.get(CHANGES_TRIE_CONFIG)
.and_then(|v| Decode::decode(&mut &v[..]).ok());
overlay.set_collect_extrinsics(changes_trie_config.is_some());
let overlay = OverlayedChanges::default();
assert!(storage.top.keys().all(|key| !is_child_storage_key(key)));
assert!(storage.children_default.keys().all(|key| is_child_storage_key(key)));
@@ -117,9 +96,7 @@ where
TestExternalities {
overlay,
offchain_db,
changes_trie_config,
extensions,
changes_trie_storage: ChangesTrieInMemoryStorage::new(),
backend: storage.into(),
storage_transaction_cache: Default::default(),
}
@@ -150,11 +127,6 @@ where
self.extensions.register(ext);
}
/// Get mutable reference to changes trie storage.
pub fn changes_trie_storage(&mut self) -> &mut ChangesTrieInMemoryStorage<H, N> {
&mut self.changes_trie_storage
}
/// Return a new backend with all pending changes.
///
/// In contrast to [`commit_all`](Self::commit_all) this will not panic if there are open
@@ -180,9 +152,8 @@ where
///
/// This will panic if there are still open transactions.
pub fn commit_all(&mut self) -> Result<(), String> {
let changes = self.overlay.drain_storage_changes::<_, _, N>(
let changes = self.overlay.drain_storage_changes::<_, _>(
&self.backend,
None,
Default::default(),
&mut Default::default(),
)?;
@@ -216,7 +187,7 @@ where
}
}
impl<H: Hasher, N: ChangesTrieBlockNumber> std::fmt::Debug for TestExternalities<H, N>
impl<H: Hasher> std::fmt::Debug for TestExternalities<H>
where
H::Out: Ord + codec::Codec,
{
@@ -225,18 +196,18 @@ where
}
}
impl<H: Hasher, N: ChangesTrieBlockNumber> PartialEq for TestExternalities<H, N>
impl<H: Hasher> PartialEq for TestExternalities<H>
where
H::Out: Ord + 'static + codec::Codec,
{
/// This doesn't test if they are in the same state, only if they contains the
/// same data at this state
fn eq(&self, other: &TestExternalities<H, N>) -> bool {
fn eq(&self, other: &TestExternalities<H>) -> bool {
self.as_backend().eq(&other.as_backend())
}
}
impl<H: Hasher, N: ChangesTrieBlockNumber> Default for TestExternalities<H, N>
impl<H: Hasher> Default for TestExternalities<H>
where
H::Out: Ord + 'static + codec::Codec,
{
@@ -245,7 +216,7 @@ where
}
}
impl<H: Hasher, N: ChangesTrieBlockNumber> From<Storage> for TestExternalities<H, N>
impl<H: Hasher> From<Storage> for TestExternalities<H>
where
H::Out: Ord + 'static + codec::Codec,
{
@@ -254,11 +225,10 @@ where
}
}
impl<H, N> sp_externalities::ExtensionStore for TestExternalities<H, N>
impl<H> sp_externalities::ExtensionStore for TestExternalities<H>
where
H: Hasher,
H::Out: Ord + codec::Codec,
N: ChangesTrieBlockNumber,
{
fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any> {
self.extensions.get_mut(type_id)
@@ -284,11 +254,10 @@ where
}
}
impl<H, N> sp_externalities::ExternalitiesExt for TestExternalities<H, N>
impl<H> sp_externalities::ExternalitiesExt for TestExternalities<H>
where
H: Hasher,
H::Out: Ord + codec::Codec,
N: ChangesTrieBlockNumber,
{
fn extension<T: Any + Extension>(&mut self) -> Option<&mut T> {
self.extension_by_type_id(TypeId::of::<T>()).and_then(<dyn Any>::downcast_mut)
@@ -312,7 +281,7 @@ mod tests {
#[test]
fn commit_should_work() {
let mut ext = TestExternalities::<BlakeTwo256, u64>::default();
let mut ext = TestExternalities::<BlakeTwo256>::default();
let mut ext = ext.ext();
ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec());
ext.set_storage(b"dog".to_vec(), b"puppy".to_vec());
@@ -324,7 +293,7 @@ mod tests {
#[test]
fn set_and_retrieve_code() {
let mut ext = TestExternalities::<BlakeTwo256, u64>::default();
let mut ext = TestExternalities::<BlakeTwo256>::default();
let mut ext = ext.ext();
let code = vec![1, 2, 3];
@@ -336,12 +305,12 @@ mod tests {
#[test]
fn check_send() {
fn assert_send<T: Send>() {}
assert_send::<TestExternalities<BlakeTwo256, u64>>();
assert_send::<TestExternalities<BlakeTwo256>>();
}
#[test]
fn commit_all_and_kill_child_storage() {
let mut ext = TestExternalities::<BlakeTwo256, u64>::default();
let mut ext = TestExternalities::<BlakeTwo256>::default();
let child_info = ChildInfo::new_default(&b"test_child"[..]);
{
@@ -366,7 +335,7 @@ mod tests {
#[test]
fn as_backend_generates_same_backend_as_commit_all() {
let mut ext = TestExternalities::<BlakeTwo256, u64>::default();
let mut ext = TestExternalities::<BlakeTwo256>::default();
{
let mut ext = ext.ext();
ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec());