pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-13 17:31:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reorganising the repository - external renames and moves (#4074)

Browse Source 
* Adding first rough ouline of the repository structure

* Remove old CI stuff

* add title

* formatting fixes

* move node-exits job's script to scripts dir

* Move docs into subdir

* move to bin

* move maintainence scripts, configs and helpers into its own dir

* add .local to ignore

* move core->client

* start up 'test' area

* move test client

* move test runtime

* make test move compile

* Add dependencies rule enforcement.

* Fix indexing.

* Update docs to reflect latest changes

* Moving /srml->/paint

* update docs

* move client/sr-* -> primitives/

* clean old readme

* remove old broken code in rhd

* update lock

* Step 1.

* starting to untangle client

* Fix after merge.

* start splitting out client interfaces

* move children and blockchain interfaces

* Move trie and state-machine to primitives.

* Fix WASM builds.

* fixing broken imports

* more interface moves

* move backend and light to interfaces

* move CallExecutor

* move cli off client

* moving around more interfaces

* re-add consensus crates into the mix

* fix subkey path

* relieve client from executor

* starting to pull out client from grandpa

* move is_decendent_of out of client

* grandpa still depends on client directly

* lemme tests pass

* rename srml->paint

* Make it compile.

* rename interfaces->client-api

* Move keyring to primitives.

* fixup libp2p dep

* fix broken use

* allow dependency enforcement to fail

* move fork-tree

* Moving wasm-builder

* make env

* move build-script-utils

* fixup broken crate depdencies and names

* fix imports for authority discovery

* fix typo

* update cargo.lock

* fixing imports

* Fix paths and add missing crates

* re-add missing crates
This commit is contained in:
Benjamin Kampmann
2019-11-14 21:51:17 +01:00
committed by Bastian Köcher
parent becc3b0a4f
commit 60e5011c72
809 changed files with 7801 additions and 6464 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/primitives/state-machine/src/backend.rs
+531
View File
@@ -0,0 +1,531 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! State machine backends. These manage the code and storage of contracts.
use std::{error, fmt, cmp::Ord, collections::HashMap, marker::PhantomData};
use log::warn;
use hash_db::Hasher;
use crate::trie_backend::TrieBackend;
use crate::trie_backend_essence::TrieBackendStorage;
use trie::{
TrieMut, MemoryDB, child_trie_root, default_child_trie_root, TrieConfiguration,
trie_types::{TrieDBMut, Layout},
};
/// A state backend is used to read state data and can have changes committed
/// to it.
///
/// The clone operation (if implemented) should be cheap.
pub trait Backend<H: Hasher>: std::fmt::Debug {
/// An error type when fetching data is not possible.
type Error: super::Error;
/// Storage changes to be applied if committing
type Transaction: Consolidate + Default;
/// Type of trie backend storage.
type TrieBackendStorage: TrieBackendStorage<H>;
/// Get keyed storage or None if there is nothing associated.
fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error>;
/// Get keyed storage value hash or None if there is nothing associated.
fn storage_hash(&self, key: &[u8]) -> Result<Option<H::Out>, Self::Error> {
self.storage(key).map(|v| v.map(|v| H::hash(&v)))
}
/// Get keyed child storage or None if there is nothing associated.
fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error>;
/// Get child keyed storage value hash or None if there is nothing associated.
fn child_storage_hash(&self, storage_key: &[u8], key: &[u8]) -> Result<Option<H::Out>, Self::Error> {
self.child_storage(storage_key, key).map(|v| v.map(|v| H::hash(&v)))
}
/// true if a key exists in storage.
fn exists_storage(&self, key: &[u8]) -> Result<bool, Self::Error> {
Ok(self.storage(key)?.is_some())
}
/// true if a key exists in child storage.
fn exists_child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result<bool, Self::Error> {
Ok(self.child_storage(storage_key, key)?.is_some())
}
/// Retrieve all entries keys of child storage and call `f` for each of those keys.
fn for_keys_in_child_storage<F: FnMut(&[u8])>(&self, storage_key: &[u8], f: F);
/// Retrieve all entries keys which start with the given prefix and
/// call `f` for each of those keys.
fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], mut f: F) {
self.for_key_values_with_prefix(prefix, |k, _v| f(k))
}
/// Retrieve all entries keys and values of which start with the given prefix and
/// call `f` for each of those keys.
fn for_key_values_with_prefix<F: FnMut(&[u8], &[u8])>(&self, prefix: &[u8], f: F);
/// Retrieve all child entries keys which start with the given prefix and
/// call `f` for each of those keys.
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(&self, storage_key: &[u8], prefix: &[u8], f: F);
/// Calculate the storage root, with given delta over what is already stored in
/// the backend, and produce a "transaction" that can be used to commit.
/// Does not include child storage updates.
fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
H::Out: Ord;
/// Calculate the child storage root, with given delta over what is already stored in
/// the backend, and produce a "transaction" that can be used to commit. The second argument
/// is true if child storage root equals default storage root.
fn child_storage_root<I>(&self, storage_key: &[u8], delta: I) -> (Vec<u8>, bool, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
H::Out: Ord;
/// Get all key/value pairs into a Vec.
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)>;
/// Get all keys with given prefix
fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
let mut all = Vec::new();
self.for_keys_with_prefix(prefix, |k| all.push(k.to_vec()));
all
}
/// Get all keys of child storage with given prefix
fn child_keys(&self, child_storage_key: &[u8], prefix: &[u8]) -> Vec<Vec<u8>> {
let mut all = Vec::new();
self.for_child_keys_with_prefix(child_storage_key, prefix, |k| all.push(k.to_vec()));
all
}
/// Try convert into trie backend.
fn as_trie_backend(&mut self) -> Option<&TrieBackend<Self::TrieBackendStorage, H>> {
None
}
/// Calculate the storage root, with given delta over what is already stored
/// in the backend, and produce a "transaction" that can be used to commit.
/// Does include child storage updates.
fn full_storage_root<I1, I2i, I2>(
&self,
delta: I1,
child_deltas: I2)
-> (H::Out, Self::Transaction)
where
I1: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
I2i: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
I2: IntoIterator<Item=(Vec<u8>, I2i)>,
<H as Hasher>::Out: Ord,
{
let mut txs: Self::Transaction = Default::default();
let mut child_roots: Vec<_> = Default::default();
// child first
for (storage_key, child_delta) in child_deltas {
let (child_root, empty, child_txs) =
self.child_storage_root(&storage_key[..], child_delta);
txs.consolidate(child_txs);
if empty {
child_roots.push((storage_key, None));
} else {
child_roots.push((storage_key, Some(child_root)));
}
}
let (root, parent_txs) = self.storage_root(
delta.into_iter().chain(child_roots.into_iter())
);
txs.consolidate(parent_txs);
(root, txs)
}
}
impl<'a, T: Backend<H>, H: Hasher> Backend<H> for &'a T {
type Error = T::Error;
type Transaction = T::Transaction;
type TrieBackendStorage = T::TrieBackendStorage;
fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
(*self).storage(key)
}
fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
(*self).child_storage(storage_key, key)
}
fn for_keys_in_child_storage<F: FnMut(&[u8])>(&self, storage_key: &[u8], f: F) {
(*self).for_keys_in_child_storage(storage_key, f)
}
fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
(*self).for_keys_with_prefix(prefix, f)
}
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(&self, storage_key: &[u8], prefix: &[u8], f: F) {
(*self).for_child_keys_with_prefix(storage_key, prefix, f)
}
fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
H::Out: Ord,
{
(*self).storage_root(delta)
}
fn child_storage_root<I>(&self, storage_key: &[u8], delta: I) -> (Vec<u8>, bool, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
H::Out: Ord,
{
(*self).child_storage_root(storage_key, delta)
}
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
(*self).pairs()
}
fn for_key_values_with_prefix<F: FnMut(&[u8], &[u8])>(&self, prefix: &[u8], f: F) {
(*self).for_key_values_with_prefix(prefix, f);
}
}
/// Trait that allows consolidate two transactions together.
pub trait Consolidate {
/// Consolidate two transactions into one.
fn consolidate(&mut self, other: Self);
}
impl Consolidate for () {
fn consolidate(&mut self, _: Self) {
()
}
}
impl Consolidate for Vec<(Option<Vec<u8>>, Vec<u8>, Option<Vec<u8>>)> {
fn consolidate(&mut self, mut other: Self) {
self.append(&mut other);
}
}
impl<H: Hasher, KF: trie::KeyFunction<H>> Consolidate for trie::GenericMemoryDB<H, KF> {
fn consolidate(&mut self, other: Self) {
trie::GenericMemoryDB::consolidate(self, other)
}
}
/// Error impossible.
// FIXME: use `!` type when stabilized. https://github.com/rust-lang/rust/issues/35121
#[derive(Debug)]
pub enum Void {}
impl fmt::Display for Void {
fn fmt(&self, _: &mut fmt::Formatter) -> fmt::Result {
match *self {}
}
}
impl error::Error for Void {
fn description(&self) -> &str { "unreachable error" }
}
/// In-memory backend. Fully recomputes tries each time `as_trie_backend` is called but useful for
/// tests and proof checking.
pub struct InMemory<H: Hasher> {
inner: HashMap<Option<Vec<u8>>, HashMap<Vec<u8>, Vec<u8>>>,
// This field is only needed for returning reference in `as_trie_backend`.
trie: Option<TrieBackend<MemoryDB<H>, H>>,
_hasher: PhantomData<H>,
}
impl<H: Hasher> std::fmt::Debug for InMemory<H> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "InMemory ({} values)", self.inner.len())
}
}
impl<H: Hasher> Default for InMemory<H> {
fn default() -> Self {
InMemory {
inner: Default::default(),
trie: None,
_hasher: PhantomData,
}
}
}
impl<H: Hasher> Clone for InMemory<H> {
fn clone(&self) -> Self {
InMemory {
inner: self.inner.clone(),
trie: None,
_hasher: PhantomData,
}
}
}
impl<H: Hasher> PartialEq for InMemory<H> {
fn eq(&self, other: &Self) -> bool {
self.inner.eq(&other.inner)
}
}
impl<H: Hasher> InMemory<H> {
/// Copy the state, with applied updates
pub fn update(&self, changes: <Self as Backend<H>>::Transaction) -> Self {
let mut inner: HashMap<_, _> = self.inner.clone();
for (storage_key, key, val) in changes {
match val {
Some(v) => { inner.entry(storage_key).or_default().insert(key, v); },
None => { inner.entry(storage_key).or_default().remove(&key); },
}
}
inner.into()
}
}
impl<H: Hasher> From<HashMap<Option<Vec<u8>>, HashMap<Vec<u8>, Vec<u8>>>> for InMemory<H> {
fn from(inner: HashMap<Option<Vec<u8>>, HashMap<Vec<u8>, Vec<u8>>>) -> Self {
InMemory {
inner: inner,
trie: None,
_hasher: PhantomData,
}
}
}
impl<H: Hasher> From<(
HashMap<Vec<u8>, Vec<u8>>,
HashMap<Vec<u8>, HashMap<Vec<u8>, Vec<u8>>>,
)> for InMemory<H> {
fn from(inners: (
HashMap<Vec<u8>, Vec<u8>>,
HashMap<Vec<u8>, HashMap<Vec<u8>, Vec<u8>>>,
)) -> Self {
let mut inner: HashMap<Option<Vec<u8>>, HashMap<Vec<u8>, Vec<u8>>>
= inners.1.into_iter().map(|(k, v)| (Some(k), v)).collect();
inner.insert(None, inners.0);
InMemory {
inner: inner,
trie: None,
_hasher: PhantomData,
}
}
}
impl<H: Hasher> From<HashMap<Vec<u8>, Vec<u8>>> for InMemory<H> {
fn from(inner: HashMap<Vec<u8>, Vec<u8>>) -> Self {
let mut expanded = HashMap::new();
expanded.insert(None, inner);
InMemory {
inner: expanded,
trie: None,
_hasher: PhantomData,
}
}
}
impl<H: Hasher> From<Vec<(Option<Vec<u8>>, Vec<u8>, Option<Vec<u8>>)>> for InMemory<H> {
fn from(inner: Vec<(Option<Vec<u8>>, Vec<u8>, Option<Vec<u8>>)>) -> Self {
let mut expanded: HashMap<Option<Vec<u8>>, HashMap<Vec<u8>, Vec<u8>>> = HashMap::new();
for (child_key, key, value) in inner {
if let Some(value) = value {
expanded.entry(child_key).or_default().insert(key, value);
}
}
expanded.into()
}
}
impl<H: Hasher> InMemory<H> {
/// child storage key iterator
pub fn child_storage_keys(&self) -> impl Iterator<Item=&[u8]> {
self.inner.iter().filter_map(|item| item.0.as_ref().map(|v|&v[..]))
}
}
impl<H: Hasher> Backend<H> for InMemory<H> {
type Error = Void;
type Transaction = Vec<(Option<Vec<u8>>, Vec<u8>, Option<Vec<u8>>)>;
type TrieBackendStorage = MemoryDB<H>;
fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
Ok(self.inner.get(&None).and_then(|map| map.get(key).map(Clone::clone)))
}
fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
Ok(self.inner.get(&Some(storage_key.to_vec())).and_then(|map| map.get(key).map(Clone::clone)))
}
fn exists_storage(&self, key: &[u8]) -> Result<bool, Self::Error> {
Ok(self.inner.get(&None).map(|map| map.get(key).is_some()).unwrap_or(false))
}
fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
self.inner.get(&None).map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f));
}
fn for_key_values_with_prefix<F: FnMut(&[u8], &[u8])>(&self, prefix: &[u8], mut f: F) {
self.inner.get(&None).map(|map| map.iter().filter(|(key, _val)| key.starts_with(prefix))
.for_each(|(k, v)| f(k, v)));
}
fn for_keys_in_child_storage<F: FnMut(&[u8])>(&self, storage_key: &[u8], mut f: F) {
self.inner.get(&Some(storage_key.to_vec())).map(|map| map.keys().for_each(|k| f(&k)));
}
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(&self, storage_key: &[u8], prefix: &[u8], f: F) {
self.inner.get(&Some(storage_key.to_vec()))
.map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f));
}
fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
<H as Hasher>::Out: Ord,
{
let existing_pairs = self.inner.get(&None)
.into_iter()
.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
let transaction: Vec<_> = delta.into_iter().collect();
let root = Layout::<H>::trie_root(existing_pairs.chain(transaction.iter().cloned())
.collect::<HashMap<_, _>>()
.into_iter()
.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
);
let full_transaction = transaction.into_iter().map(|(k, v)| (None, k, v)).collect();
(root, full_transaction)
}
fn child_storage_root<I>(&self, storage_key: &[u8], delta: I) -> (Vec<u8>, bool, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
H::Out: Ord
{
let storage_key = storage_key.to_vec();
let existing_pairs = self.inner.get(&Some(storage_key.clone()))
.into_iter()
.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
let transaction: Vec<_> = delta.into_iter().collect();
let root = child_trie_root::<Layout<H>, _, _, _>(
&storage_key,
existing_pairs.chain(transaction.iter().cloned())
.collect::<HashMap<_, _>>()
.into_iter()
.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
);
let full_transaction = transaction.into_iter().map(|(k, v)| (Some(storage_key.clone()), k, v)).collect();
let is_default = root == default_child_trie_root::<Layout<H>>(&storage_key);
(root, is_default, full_transaction)
}
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
self.inner.get(&None)
.into_iter()
.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone())))
.collect()
}
fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
self.inner.get(&None)
.into_iter()
.flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned())
.collect()
}
fn child_keys(&self, storage_key: &[u8], prefix: &[u8]) -> Vec<Vec<u8>> {
self.inner.get(&Some(storage_key.to_vec()))
.into_iter()
.flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned())
.collect()
}
fn as_trie_backend(&mut self)-> Option<&TrieBackend<Self::TrieBackendStorage, H>> {
let mut mdb = MemoryDB::default();
let mut new_child_roots = Vec::new();
let mut root_map = None;
for (storage_key, map) in &self.inner {
if let Some(storage_key) = storage_key.as_ref() {
let ch = insert_into_memory_db::<H, _>(&mut mdb, map.clone().into_iter())?;
new_child_roots.push((storage_key.clone(), ch.as_ref().into()));
} else {
root_map = Some(map);
}
}
let root = match root_map {
Some(map) => insert_into_memory_db::<H, _>(
&mut mdb,
map.clone().into_iter().chain(new_child_roots.into_iter()),
)?,
None => insert_into_memory_db::<H, _>(
&mut mdb,
new_child_roots.into_iter(),
)?,
};
self.trie = Some(TrieBackend::new(mdb, root));
self.trie.as_ref()
}
}
/// Insert input pairs into memory db.
pub(crate) fn insert_into_memory_db<H, I>(mdb: &mut MemoryDB<H>, input: I) -> Option<H::Out>
where
H: Hasher,
I: IntoIterator<Item=(Vec<u8>, Vec<u8>)>,
{
let mut root = <H as Hasher>::Out::default();
{
let mut trie = TrieDBMut::<H>::new(mdb, &mut root);
for (key, value) in input {
if let Err(e) = trie.insert(&key, &value) {
warn!(target: "trie", "Failed to write to trie: {}", e);
return None;
}
}
}
Some(root)
}
#[cfg(test)]
mod tests {
use super::*;
/// Assert in memory backend with only child trie keys works as trie backend.
#[test]
fn in_memory_with_child_trie_only() {
let storage = InMemory::<primitives::Blake2Hasher>::default();
let mut storage = storage.update(
vec![(Some(b"1".to_vec()), b"2".to_vec(), Some(b"3".to_vec()))]
);
let trie_backend = storage.as_trie_backend().unwrap();
assert_eq!(trie_backend.child_storage(b"1", b"2").unwrap(), Some(b"3".to_vec()));
assert!(trie_backend.storage(b"1").unwrap().is_some());
}
}
substrate/primitives/state-machine/src/basic.rs
+308
View File
@@ -0,0 +1,308 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Basic implementation for Externalities.
use std::{collections::HashMap, any::{TypeId, Any}, iter::FromIterator};
use crate::backend::{Backend, InMemory};
use hash_db::Hasher;
use trie::{TrieConfiguration, default_child_trie_root};
use trie::trie_types::Layout;
use primitives::{
storage::{
well_known_keys::is_child_storage_key, ChildStorageKey, StorageOverlay,
ChildrenStorageOverlay
},
traits::Externalities, Blake2Hasher, hash::H256,
};
use log::warn;
/// Simple HashMap-based Externalities impl.
#[derive(Debug)]
pub struct BasicExternalities {
top: StorageOverlay,
children: ChildrenStorageOverlay,
}
impl BasicExternalities {
/// Create a new instance of `BasicExternalities`
pub fn new(top: StorageOverlay, children: ChildrenStorageOverlay) -> Self {
BasicExternalities {
top,
children,
}
}
/// Insert key/value
pub fn insert(&mut self, k: Vec<u8>, v: Vec<u8>) -> Option<Vec<u8>> {
self.top.insert(k, v)
}
/// Consume self and returns inner storages
pub fn into_storages(self) -> (
HashMap<Vec<u8>, Vec<u8>>,
HashMap<Vec<u8>, HashMap<Vec<u8>, Vec<u8>>>,
) {
(self.top, self.children)
}
/// Execute the given closure `f` with the externalities set and initialized with `storage`.
///
/// Returns the result of the closure and updates `storage` with all changes.
pub fn execute_with_storage<R>(
storage: &mut (StorageOverlay, ChildrenStorageOverlay),
f: impl FnOnce() -> R,
) -> R {
let mut ext = Self {
top: storage.0.drain().collect(),
children: storage.1.drain().collect(),
};
let r = ext.execute_with(f);
*storage = ext.into_storages();
r
}
/// Execute the given closure while `self` is set as externalities.
///
/// Returns the result of the given closure.
pub fn execute_with<R>(&mut self, f: impl FnOnce() -> R) -> R {
externalities::set_and_run_with_externalities(self, f)
}
}
impl PartialEq for BasicExternalities {
fn eq(&self, other: &BasicExternalities) -> bool {
self.top.eq(&other.top) && self.children.eq(&other.children)
}
}
impl FromIterator<(Vec<u8>, Vec<u8>)> for BasicExternalities {
fn from_iter<I: IntoIterator<Item=(Vec<u8>, Vec<u8>)>>(iter: I) -> Self {
let mut t = Self::default();
t.top.extend(iter);
t
}
}
impl Default for BasicExternalities {
fn default() -> Self { Self::new(Default::default(), Default::default()) }
}
impl From<HashMap<Vec<u8>, Vec<u8>>> for BasicExternalities {
fn from(hashmap: HashMap<Vec<u8>, Vec<u8>>) -> Self {
BasicExternalities {
top: hashmap,
children: Default::default(),
}
}
}
impl Externalities for BasicExternalities {
fn storage(&self, key: &[u8]) -> Option<Vec<u8>> {
self.top.get(key).cloned()
}
fn storage_hash(&self, key: &[u8]) -> Option<H256> {
self.storage(key).map(|v| Blake2Hasher::hash(&v))
}
fn original_storage(&self, key: &[u8]) -> Option<Vec<u8>> {
self.storage(key)
}
fn original_storage_hash(&self, key: &[u8]) -> Option<H256> {
self.storage_hash(key)
}
fn child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option<Vec<u8>> {
self.children.get(storage_key.as_ref()).and_then(|child| child.get(key)).cloned()
}
fn child_storage_hash(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option<H256> {
self.child_storage(storage_key, key).map(|v| Blake2Hasher::hash(&v))
}
fn original_child_storage_hash(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option<H256> {
self.child_storage_hash(storage_key, key)
}
fn original_child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option<Vec<u8>> {
Externalities::child_storage(self, storage_key, key)
}
fn place_storage(&mut self, key: Vec<u8>, maybe_value: Option<Vec<u8>>) {
if is_child_storage_key(&key) {
warn!(target: "trie", "Refuse to set child storage key via main storage");
return;
}
match maybe_value {
Some(value) => { self.top.insert(key, value); }
None => { self.top.remove(&key); }
}
}
fn place_child_storage(
&mut self,
storage_key: ChildStorageKey,
key: Vec<u8>,
value: Option<Vec<u8>>,
) {
let child_map = self.children.entry(storage_key.into_owned()).or_default();
if let Some(value) = value {
child_map.insert(key, value);
} else {
child_map.remove(&key);
}
}
fn kill_child_storage(&mut self, storage_key: ChildStorageKey) {
self.children.remove(storage_key.as_ref());
}
fn clear_prefix(&mut self, prefix: &[u8]) {
if is_child_storage_key(prefix) {
warn!(
target: "trie",
"Refuse to clear prefix that is part of child storage key via main storage"
);
return;
}
self.top.retain(|key, _| !key.starts_with(prefix));
}
fn clear_child_prefix(&mut self, storage_key: ChildStorageKey, prefix: &[u8]) {
if let Some(child) = self.children.get_mut(storage_key.as_ref()) {
child.retain(|key, _| !key.starts_with(prefix));
}
}
fn chain_id(&self) -> u64 { 42 }
fn storage_root(&mut self) -> H256 {
let mut top = self.top.clone();
let keys: Vec<_> = self.children.keys().map(|k| k.to_vec()).collect();
// Single child trie implementation currently allows using the same child
// empty root for all child trie. Using null storage key until multiple
// type of child trie support.
let empty_hash = default_child_trie_root::<Layout<Blake2Hasher>>(&[]);
for storage_key in keys {
let child_root = self.child_storage_root(
ChildStorageKey::from_slice(storage_key.as_slice())
.expect("Map only feed by valid keys; qed"),
);
if &empty_hash[..] == &child_root[..] {
top.remove(&storage_key);
} else {
top.insert(storage_key, child_root);
}
}
Layout::<Blake2Hasher>::trie_root(self.top.clone())
}
fn child_storage_root(&mut self, storage_key: ChildStorageKey) -> Vec<u8> {
if let Some(child) = self.children.get(storage_key.as_ref()) {
let delta = child.clone().into_iter().map(|(k, v)| (k, Some(v)));
InMemory::<Blake2Hasher>::default().child_storage_root(storage_key.as_ref(), delta).0
} else {
default_child_trie_root::<Layout<Blake2Hasher>>(storage_key.as_ref())
}
}
fn storage_changes_root(&mut self, _parent: H256) -> Result<Option<H256>, ()> {
Ok(None)
}
}
impl externalities::ExtensionStore for BasicExternalities {
fn extension_by_type_id(&mut self, _: TypeId) -> Option<&mut dyn Any> {
warn!("Extensions are not supported by `BasicExternalities`.");
None
}
}
#[cfg(test)]
mod tests {
use super::*;
use primitives::{H256, map};
use primitives::storage::well_known_keys::CODE;
use hex_literal::hex;
#[test]
fn commit_should_work() {
let mut ext = BasicExternalities::default();
ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec());
ext.set_storage(b"dog".to_vec(), b"puppy".to_vec());
ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec());
const ROOT: [u8; 32] = hex!("39245109cef3758c2eed2ccba8d9b370a917850af3824bc8348d505df2c298fa");
assert_eq!(ext.storage_root(), H256::from(ROOT));
}
#[test]
fn set_and_retrieve_code() {
let mut ext = BasicExternalities::default();
let code = vec![1, 2, 3];
ext.set_storage(CODE.to_vec(), code.clone());
assert_eq!(&ext.storage(CODE).unwrap(), &code);
}
#[test]
fn children_works() {
let child_storage = b":child_storage:default:test".to_vec();
let mut ext = BasicExternalities::new(
Default::default(),
map![
child_storage.clone() => map![
b"doe".to_vec() => b"reindeer".to_vec()
]
]
);
let child = || ChildStorageKey::from_vec(child_storage.clone()).unwrap();
assert_eq!(ext.child_storage(child(), b"doe"), Some(b"reindeer".to_vec()));
ext.set_child_storage(child(), b"dog".to_vec(), b"puppy".to_vec());
assert_eq!(ext.child_storage(child(), b"dog"), Some(b"puppy".to_vec()));
ext.clear_child_storage(child(), b"dog");
assert_eq!(ext.child_storage(child(), b"dog"), None);
ext.kill_child_storage(child());
assert_eq!(ext.child_storage(child(), b"doe"), None);
}
#[test]
fn basic_externalities_is_empty() {
// Make sure no values are set by default in `BasicExternalities`.
let (storage, child_storage) = BasicExternalities::new(
Default::default(),
Default::default(),
).into_storages();
assert!(storage.is_empty());
assert!(child_storage.is_empty());
}
}
substrate/primitives/state-machine/src/changes_trie/build.rs
+762
View File
@@ -0,0 +1,762 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Structures and functions required to build changes trie for given block.
use std::collections::{BTreeMap, BTreeSet};
use std::collections::btree_map::Entry;
use codec::Decode;
use hash_db::Hasher;
use num_traits::One;
use crate::backend::Backend;
use crate::overlayed_changes::OverlayedChanges;
use crate::trie_backend_essence::TrieBackendEssence;
use crate::changes_trie::build_iterator::digest_build_iterator;
use crate::changes_trie::input::{InputKey, InputPair, DigestIndex, ExtrinsicIndex};
use crate::changes_trie::{AnchorBlockId, ConfigurationRange, Storage, BlockNumber};
use crate::changes_trie::input::ChildIndex;
/// Prepare input pairs for building a changes trie of given block.
///
/// Returns Err if storage error has occurred OR if storage haven't returned
/// required data.
pub(crate) fn prepare_input<'a, B, H, Number>(
backend: &'a B,
storage: &'a dyn Storage<H, Number>,
config: ConfigurationRange<'a, Number>,
changes: &'a OverlayedChanges,
parent: &'a AnchorBlockId<H::Out, Number>,
) -> Result<(
impl Iterator<Item=InputPair<Number>> + 'a,
Vec<(ChildIndex<Number>, impl Iterator<Item=InputPair<Number>> + 'a)>,
Vec<Number>,
), String>
where
B: Backend<H>,
H: Hasher + 'a,
Number: BlockNumber,
{
let number = parent.number.clone() + One::one();
let (extrinsics_input, children_extrinsics_input) = prepare_extrinsics_input(
backend,
&number,
changes,
)?;
let (digest_input, mut children_digest_input, digest_input_blocks) = prepare_digest_input::<H, Number>(
parent,
config,
number,
storage,
)?;
let mut children_digest = Vec::with_capacity(children_extrinsics_input.len());
for (child_index, ext_iter) in children_extrinsics_input.into_iter() {
let dig_iter = children_digest_input.remove(&child_index);
children_digest.push((
child_index,
Some(ext_iter).into_iter().flatten()
.chain(dig_iter.into_iter().flatten()),
));
}
for (child_index, dig_iter) in children_digest_input.into_iter() {
children_digest.push((
child_index,
None.into_iter().flatten()
.chain(Some(dig_iter).into_iter().flatten()),
));
}
Ok((
extrinsics_input.chain(digest_input),
children_digest,
digest_input_blocks,
))
}
/// Prepare ExtrinsicIndex input pairs.
fn prepare_extrinsics_input<'a, B, H, Number>(
backend: &'a B,
block: &Number,
changes: &'a OverlayedChanges,
) -> Result<(
impl Iterator<Item=InputPair<Number>> + 'a,
BTreeMap<ChildIndex<Number>, impl Iterator<Item=InputPair<Number>> + 'a>,
), String>
where
B: Backend<H>,
H: Hasher + 'a,
Number: BlockNumber,
{
let mut children_keys = BTreeSet::<Vec<u8>>::new();
let mut children_result = BTreeMap::new();
for (storage_key, _) in changes.prospective.children.iter()
.chain(changes.committed.children.iter()) {
children_keys.insert(storage_key.clone());
}
for storage_key in children_keys {
let child_index = ChildIndex::<Number> {
block: block.clone(),
storage_key: storage_key.clone(),
};
let iter = prepare_extrinsics_input_inner(backend, block, changes, Some(storage_key))?;
children_result.insert(child_index, iter);
}
let top = prepare_extrinsics_input_inner(backend, block, changes, None)?;
Ok((top, children_result))
}
fn prepare_extrinsics_input_inner<'a, B, H, Number>(
backend: &'a B,
block: &Number,
changes: &'a OverlayedChanges,
storage_key: Option<Vec<u8>>,
) -> Result<impl Iterator<Item=InputPair<Number>> + 'a, String>
where
B: Backend<H>,
H: Hasher,
Number: BlockNumber,
{
let (committed, prospective) = if let Some(sk) = storage_key.as_ref() {
(changes.committed.children.get(sk), changes.prospective.children.get(sk))
} else {
(Some(&changes.committed.top), Some(&changes.prospective.top))
};
committed.iter().flat_map(|c| c.iter())
.chain(prospective.iter().flat_map(|c| c.iter()))
.filter(|( _, v)| v.extrinsics.is_some())
.try_fold(BTreeMap::new(), |mut map: BTreeMap<&[u8], (ExtrinsicIndex<Number>, Vec<u32>)>, (k, v)| {
match map.entry(k) {
Entry::Vacant(entry) => {
// ignore temporary values (values that have null value at the end of operation
// AND are not in storage at the beginning of operation
if let Some(sk) = storage_key.as_ref() {
if !changes.child_storage(sk, k).map(|v| v.is_some()).unwrap_or_default() {
if !backend.exists_child_storage(sk, k).map_err(|e| format!("{}", e))? {
return Ok(map);
}
}
} else {
if !changes.storage(k).map(|v| v.is_some()).unwrap_or_default() {
if !backend.exists_storage(k).map_err(|e| format!("{}", e))? {
return Ok(map);
}
}
};
let extrinsics = v.extrinsics.as_ref()
.expect("filtered by filter() call above; qed")
.iter().cloned().collect();
entry.insert((ExtrinsicIndex {
block: block.clone(),
key: k.to_vec(),
}, extrinsics));
},
Entry::Occupied(mut entry) => {
// we do not need to check for temporary values here, because entry is Occupied
// AND we are checking it before insertion
let extrinsics = &mut entry.get_mut().1;
extrinsics.extend(
v.extrinsics.as_ref()
.expect("filtered by filter() call above; qed")
.iter()
.cloned()
);
extrinsics.sort_unstable();
},
}
Ok(map)
})
.map(|pairs| pairs.into_iter().map(|(_, (k, v))| InputPair::ExtrinsicIndex(k, v)))
}
/// Prepare DigestIndex input pairs.
fn prepare_digest_input<'a, H, Number>(
parent: &'a AnchorBlockId<H::Out, Number>,
config: ConfigurationRange<Number>,
block: Number,
storage: &'a dyn Storage<H, Number>,
) -> Result<(
impl Iterator<Item=InputPair<Number>> + 'a,
BTreeMap<ChildIndex<Number>, impl Iterator<Item=InputPair<Number>> + 'a>,
Vec<Number>,
), String>
where
H: Hasher,
H::Out: 'a,
Number: BlockNumber,
{
let build_skewed_digest = config.end.as_ref() == Some(&block);
let block_for_digest = if build_skewed_digest {
config.config.next_max_level_digest_range(config.zero.clone(), block.clone())
.map(|(_, end)| end)
.unwrap_or_else(|| block.clone())
} else {
block.clone()
};
let digest_input_blocks = digest_build_iterator(config, block_for_digest).collect::<Vec<_>>();
digest_input_blocks.clone().into_iter()
.try_fold(
(BTreeMap::new(), BTreeMap::new()), move |(mut map, mut child_map), digest_build_block| {
let extrinsic_prefix = ExtrinsicIndex::key_neutral_prefix(digest_build_block.clone());
let digest_prefix = DigestIndex::key_neutral_prefix(digest_build_block.clone());
let child_prefix = ChildIndex::key_neutral_prefix(digest_build_block.clone());
let trie_root = storage.root(parent, digest_build_block.clone())?;
let trie_root = trie_root.ok_or_else(|| format!("No changes trie root for block {}", digest_build_block.clone()))?;
let insert_to_map = |map: &mut BTreeMap<_,_>, key: Vec<u8>| {
match map.entry(key.clone()) {
Entry::Vacant(entry) => {
entry.insert((DigestIndex {
block: block.clone(),
key,
}, vec![digest_build_block.clone()]));
},
Entry::Occupied(mut entry) => {
// DigestIndexValue must be sorted. Here we are relying on the fact that digest_build_iterator()
// returns blocks in ascending order => we only need to check for duplicates
//
// is_dup_block could be true when key has been changed in both digest block
// AND other blocks that it covers
let is_dup_block = entry.get().1.last() == Some(&digest_build_block);
if !is_dup_block {
entry.get_mut().1.push(digest_build_block.clone());
}
},
}
};
// try to get all updated keys from cache
let populated_from_cache = storage.with_cached_changed_keys(
&trie_root,
&mut |changed_keys| {
for (storage_key, changed_keys) in changed_keys {
let map = match storage_key {
Some(storage_key) => child_map
.entry(ChildIndex::<Number> {
block: block.clone(),
storage_key: storage_key.clone(),
})
.or_default(),
None => &mut map,
};
for changed_key in changed_keys.iter().cloned() {
insert_to_map(map, changed_key);
}
}
}
);
if populated_from_cache {
return Ok((map, child_map));
}
let mut children_roots = BTreeMap::<Vec<u8>, _>::new();
{
let trie_storage = TrieBackendEssence::<_, H>::new(
crate::changes_trie::TrieBackendStorageAdapter(storage),
trie_root,
);
trie_storage.for_key_values_with_prefix(&child_prefix, |key, value|
if let Ok(InputKey::ChildIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
if let Ok(value) = <Vec<u8>>::decode(&mut &value[..]) {
let mut trie_root = <H as Hasher>::Out::default();
trie_root.as_mut().copy_from_slice(&value[..]);
children_roots.insert(trie_key.storage_key, trie_root);
}
});
trie_storage.for_keys_with_prefix(&extrinsic_prefix, |key|
if let Ok(InputKey::ExtrinsicIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
insert_to_map(&mut map, trie_key.key);
});
trie_storage.for_keys_with_prefix(&digest_prefix, |key|
if let Ok(InputKey::DigestIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
insert_to_map(&mut map, trie_key.key);
});
}
for (storage_key, trie_root) in children_roots.into_iter() {
let child_index = ChildIndex::<Number> {
block: block.clone(),
storage_key,
};
let mut map = child_map.entry(child_index).or_default();
let trie_storage = TrieBackendEssence::<_, H>::new(
crate::changes_trie::TrieBackendStorageAdapter(storage),
trie_root,
);
trie_storage.for_keys_with_prefix(&extrinsic_prefix, |key|
if let Ok(InputKey::ExtrinsicIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
insert_to_map(&mut map, trie_key.key);
});
trie_storage.for_keys_with_prefix(&digest_prefix, |key|
if let Ok(InputKey::DigestIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
insert_to_map(&mut map, trie_key.key);
});
}
Ok((map, child_map))
})
.map(|(pairs, child_pairs)| (
pairs.into_iter().map(|(_, (k, v))| InputPair::DigestIndex(k, v)),
child_pairs.into_iter().map(|(sk, pairs)|
(sk, pairs.into_iter().map(|(_, (k, v))| InputPair::DigestIndex(k, v)))).collect(),
digest_input_blocks,
))
}
#[cfg(test)]
mod test {
use codec::Encode;
use primitives::Blake2Hasher;
use primitives::storage::well_known_keys::{EXTRINSIC_INDEX};
use crate::backend::InMemory;
use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage};
use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData};
use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet};
use super::*;
fn prepare_for_build(zero: u64) -> (
InMemory<Blake2Hasher>,
InMemoryStorage<Blake2Hasher, u64>,
OverlayedChanges,
Configuration,
) {
let config = Configuration { digest_interval: 4, digest_levels: 2 };
let backend: InMemory<_> = vec![
(vec![100], vec![255]),
(vec![101], vec![255]),
(vec![102], vec![255]),
(vec![103], vec![255]),
(vec![104], vec![255]),
(vec![105], vec![255]),
].into_iter().collect::<::std::collections::HashMap<_, _>>().into();
let child_trie_key1 = b"1".to_vec();
let child_trie_key2 = b"2".to_vec();
let storage = InMemoryStorage::with_inputs(vec![
(zero + 1, vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![100] }, vec![1, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![101] }, vec![0, 2]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![105] }, vec![0, 2, 4]),
]),
(zero + 2, vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 2, key: vec![102] }, vec![0]),
]),
(zero + 3, vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 3, key: vec![100] }, vec![0]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 3, key: vec![105] }, vec![1]),
]),
(zero + 4, vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![101] }, vec![1]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![103] }, vec![0, 1]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![100] }, vec![zero + 1, zero + 3]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![101] }, vec![zero + 1]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![102] }, vec![zero + 2]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![105] }, vec![zero + 1, zero + 3]),
]),
(zero + 5, Vec::new()),
(zero + 6, vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 6, key: vec![105] }, vec![2]),
]),
(zero + 7, Vec::new()),
(zero + 8, vec![
InputPair::DigestIndex(DigestIndex { block: zero + 8, key: vec![105] }, vec![zero + 6]),
]),
(zero + 9, Vec::new()), (zero + 10, Vec::new()), (zero + 11, Vec::new()), (zero + 12, Vec::new()),
(zero + 13, Vec::new()), (zero + 14, Vec::new()), (zero + 15, Vec::new()),
], vec![(child_trie_key1.clone(), vec![
(zero + 1, vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![100] }, vec![1, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![101] }, vec![0, 2]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![105] }, vec![0, 2, 4]),
]),
(zero + 2, vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 2, key: vec![102] }, vec![0]),
]),
(zero + 4, vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 2, key: vec![102] }, vec![0, 3]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![102] }, vec![zero + 2]),
]),
]),
]);
let changes = OverlayedChanges {
prospective: OverlayedChangeSet { top: vec![
(vec![100], OverlayedValue {
value: Some(vec![200]),
extrinsics: Some(vec![0, 2].into_iter().collect())
}),
(vec![103], OverlayedValue {
value: None,
extrinsics: Some(vec![0, 1].into_iter().collect())
}),
].into_iter().collect(),
children: vec![
(child_trie_key1.clone(), vec![
(vec![100], OverlayedValue {
value: Some(vec![200]),
extrinsics: Some(vec![0, 2].into_iter().collect())
})
].into_iter().collect()),
(child_trie_key2, vec![
(vec![100], OverlayedValue {
value: Some(vec![200]),
extrinsics: Some(vec![0, 2].into_iter().collect())
})
].into_iter().collect()),
].into_iter().collect()
},
committed: OverlayedChangeSet { top: vec![
(EXTRINSIC_INDEX.to_vec(), OverlayedValue {
value: Some(3u32.encode()),
extrinsics: None,
}),
(vec![100], OverlayedValue {
value: Some(vec![202]),
extrinsics: Some(vec![3].into_iter().collect())
}),
(vec![101], OverlayedValue {
value: Some(vec![203]),
extrinsics: Some(vec![1].into_iter().collect())
}),
].into_iter().collect(),
children: vec![
(child_trie_key1, vec![
(vec![100], OverlayedValue {
value: Some(vec![202]),
extrinsics: Some(vec![3].into_iter().collect())
})
].into_iter().collect()),
].into_iter().collect(),
},
changes_trie_config: Some(config.clone()),
};
(backend, storage, changes, config)
}
fn configuration_range<'a>(config: &'a Configuration, zero: u64) -> ConfigurationRange<'a, u64> {
ConfigurationRange {
config,
zero,
end: None,
}
}
#[test]
fn build_changes_trie_nodes_on_non_digest_block() {
fn test_with_zero(zero: u64) {
let (backend, storage, changes, config) = prepare_for_build(zero);
let parent = AnchorBlockId { hash: Default::default(), number: zero + 4 };
let changes_trie_nodes = prepare_input(
&backend,
&storage,
configuration_range(&config, zero),
&changes,
&parent,
).unwrap();
assert_eq!(changes_trie_nodes.0.collect::<Vec<InputPair<u64>>>(), vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 5, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 5, key: vec![101] }, vec![1]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 5, key: vec![103] }, vec![0, 1]),
]);
assert_eq!(changes_trie_nodes.1.into_iter()
.map(|(k,v)| (k, v.collect::<Vec<_>>())).collect::<Vec<_>>(), vec![
(ChildIndex { block: zero + 5u64, storage_key: b"1".to_vec() },
vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 5u64, key: vec![100] }, vec![0, 2, 3]),
]),
(ChildIndex { block: zero + 5, storage_key: b"2".to_vec() },
vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 5, key: vec![100] }, vec![0, 2]),
]),
]);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn build_changes_trie_nodes_on_digest_block_l1() {
fn test_with_zero(zero: u64) {
let (backend, storage, changes, config) = prepare_for_build(zero);
let parent = AnchorBlockId { hash: Default::default(), number: zero + 3 };
let changes_trie_nodes = prepare_input(
&backend,
&storage,
configuration_range(&config, zero),
&changes,
&parent,
).unwrap();
assert_eq!(changes_trie_nodes.0.collect::<Vec<InputPair<u64>>>(), vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![101] }, vec![1]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![103] }, vec![0, 1]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![100] }, vec![zero + 1, zero + 3]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![101] }, vec![zero + 1]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![102] }, vec![zero + 2]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![105] }, vec![zero + 1, zero + 3]),
]);
assert_eq!(changes_trie_nodes.1.into_iter()
.map(|(k,v)| (k, v.collect::<Vec<_>>())).collect::<Vec<_>>(), vec![
(ChildIndex { block: zero + 4u64, storage_key: b"1".to_vec() },
vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4u64, key: vec![100] }, vec![0, 2, 3]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![100] }, vec![zero + 1]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![101] }, vec![zero + 1]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![102] }, vec![zero + 2]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![105] }, vec![zero + 1]),
]),
(ChildIndex { block: zero + 4, storage_key: b"2".to_vec() },
vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![100] }, vec![0, 2]),
]),
]);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn build_changes_trie_nodes_on_digest_block_l2() {
fn test_with_zero(zero: u64) {
let (backend, storage, changes, config) = prepare_for_build(zero);
let parent = AnchorBlockId { hash: Default::default(), number: zero + 15 };
let changes_trie_nodes = prepare_input(
&backend,
&storage,
configuration_range(&config, zero),
&changes,
&parent,
).unwrap();
assert_eq!(changes_trie_nodes.0.collect::<Vec<InputPair<u64>>>(), vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 16, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 16, key: vec![101] }, vec![1]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 16, key: vec![103] }, vec![0, 1]),
InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![100] }, vec![zero + 4]),
InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![101] }, vec![zero + 4]),
InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![102] }, vec![zero + 4]),
InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![103] }, vec![zero + 4]),
InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![105] }, vec![zero + 4, zero + 8]),
]);
assert_eq!(changes_trie_nodes.1.into_iter()
.map(|(k,v)| (k, v.collect::<Vec<_>>())).collect::<Vec<_>>(), vec![
(ChildIndex { block: zero + 16u64, storage_key: b"1".to_vec() },
vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 16u64, key: vec![100] }, vec![0, 2, 3]),
InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![102] }, vec![zero + 4]),
]),
(ChildIndex { block: zero + 16, storage_key: b"2".to_vec() },
vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 16, key: vec![100] }, vec![0, 2]),
]),
]);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn build_changes_trie_nodes_on_skewed_digest_block() {
fn test_with_zero(zero: u64) {
let (backend, storage, changes, config) = prepare_for_build(zero);
let parent = AnchorBlockId { hash: Default::default(), number: zero + 10 };
let mut configuration_range = configuration_range(&config, zero);
let changes_trie_nodes = prepare_input(
&backend,
&storage,
configuration_range.clone(),
&changes,
&parent,
).unwrap();
assert_eq!(changes_trie_nodes.0.collect::<Vec<InputPair<u64>>>(), vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![101] }, vec![1]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![103] }, vec![0, 1]),
]);
configuration_range.end = Some(zero + 11);
let changes_trie_nodes = prepare_input(
&backend,
&storage,
configuration_range,
&changes,
&parent,
).unwrap();
assert_eq!(changes_trie_nodes.0.collect::<Vec<InputPair<u64>>>(), vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![101] }, vec![1]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![103] }, vec![0, 1]),
InputPair::DigestIndex(DigestIndex { block: zero + 11, key: vec![100] }, vec![zero + 4]),
InputPair::DigestIndex(DigestIndex { block: zero + 11, key: vec![101] }, vec![zero + 4]),
InputPair::DigestIndex(DigestIndex { block: zero + 11, key: vec![102] }, vec![zero + 4]),
InputPair::DigestIndex(DigestIndex { block: zero + 11, key: vec![103] }, vec![zero + 4]),
InputPair::DigestIndex(DigestIndex { block: zero + 11, key: vec![105] }, vec![zero + 4, zero + 8]),
]);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn build_changes_trie_nodes_ignores_temporary_storage_values() {
fn test_with_zero(zero: u64) {
let (backend, storage, mut changes, config) = prepare_for_build(zero);
// 110: missing from backend, set to None in overlay
changes.prospective.top.insert(vec![110], OverlayedValue {
value: None,
extrinsics: Some(vec![1].into_iter().collect())
});
let parent = AnchorBlockId { hash: Default::default(), number: zero + 3 };
let changes_trie_nodes = prepare_input(
&backend,
&storage,
configuration_range(&config, zero),
&changes,
&parent,
).unwrap();
assert_eq!(changes_trie_nodes.0.collect::<Vec<InputPair<u64>>>(), vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![101] }, vec![1]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![103] }, vec![0, 1]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![100] }, vec![zero + 1, zero + 3]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![101] }, vec![zero + 1]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![102] }, vec![zero + 2]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![105] }, vec![zero + 1, zero + 3]),
]);
assert_eq!(changes_trie_nodes.1.into_iter()
.map(|(k,v)| (k, v.collect::<Vec<_>>())).collect::<Vec<_>>(), vec![
(ChildIndex { block: zero + 4u64, storage_key: b"1".to_vec() },
vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4u64, key: vec![100] }, vec![0, 2, 3]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![100] }, vec![zero + 1]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![101] }, vec![zero + 1]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![102] }, vec![zero + 2]),
InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![105] }, vec![zero + 1]),
]),
(ChildIndex { block: zero + 4, storage_key: b"2".to_vec() },
vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![100] }, vec![0, 2]),
]),
]);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn cache_is_used_when_changes_trie_is_built() {
let (backend, mut storage, changes, _) = prepare_for_build(0);
let config = changes.changes_trie_config.as_ref().unwrap();
let parent = AnchorBlockId { hash: Default::default(), number: 15 };
// override some actual values from storage with values from the cache
//
// top-level storage:
// (keys 100, 101, 103, 105 are now missing from block#4 => they do not appear
// in l2 digest at block 16)
//
// "1" child storage:
// key 102 is now missing from block#4 => it doesn't appear in l2 digest at block 16
// (keys 103, 104) are now added to block#4 => they appear in l2 digest at block 16
//
// "2" child storage:
// (keys 105, 106) are now added to block#4 => they appear in l2 digest at block 16
let trie_root4 = storage.root(&parent, 4).unwrap().unwrap();
let cached_data4 = IncompleteCacheAction::CacheBuildData(IncompleteCachedBuildData::new())
.set_digest_input_blocks(vec![1, 2, 3])
.insert(None, vec![vec![100], vec![102]].into_iter().collect())
.insert(Some(b"1".to_vec()), vec![vec![103], vec![104]].into_iter().collect())
.insert(Some(b"2".to_vec()), vec![vec![105], vec![106]].into_iter().collect())
.complete(4, &trie_root4);
storage.cache_mut().perform(cached_data4);
let (root_changes_trie_nodes, child_changes_tries_nodes, _) = prepare_input(
&backend,
&storage,
configuration_range(&config, 0),
&changes,
&parent,
).unwrap();
assert_eq!(root_changes_trie_nodes.collect::<Vec<InputPair<u64>>>(), vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![101] }, vec![1]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![103] }, vec![0, 1]),
InputPair::DigestIndex(DigestIndex { block: 16, key: vec![100] }, vec![4]),
InputPair::DigestIndex(DigestIndex { block: 16, key: vec![102] }, vec![4]),
InputPair::DigestIndex(DigestIndex { block: 16, key: vec![105] }, vec![8]),
]);
let child_changes_tries_nodes = child_changes_tries_nodes
.into_iter()
.map(|(k, i)| (k, i.collect::<Vec<_>>()))
.collect::<BTreeMap<_, _>>();
assert_eq!(
child_changes_tries_nodes.get(&ChildIndex { block: 16u64, storage_key: b"1".to_vec() }).unwrap(),
&vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16u64, key: vec![100] }, vec![0, 2, 3]),
InputPair::DigestIndex(DigestIndex { block: 16u64, key: vec![103] }, vec![4]),
InputPair::DigestIndex(DigestIndex { block: 16u64, key: vec![104] }, vec![4]),
],
);
assert_eq!(
child_changes_tries_nodes.get(&ChildIndex { block: 16u64, storage_key: b"2".to_vec() }).unwrap(),
&vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16u64, key: vec![100] }, vec![0, 2]),
InputPair::DigestIndex(DigestIndex { block: 16u64, key: vec![105] }, vec![4]),
InputPair::DigestIndex(DigestIndex { block: 16u64, key: vec![106] }, vec![4]),
],
);
}
}
substrate/primitives/state-machine/src/changes_trie/build_cache.rs
+262
View File
@@ -0,0 +1,262 @@
// Copyright 2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Changes tries build cache.
use std::collections::{HashMap, HashSet};
/// Changes trie build cache.
///
/// Helps to avoid read of changes tries from the database when digest trie
/// is built. It holds changed keys for every block (indexed by changes trie
/// root) that could be referenced by future digest items. For digest entries
/// it also holds keys covered by this digest. Entries for top level digests
/// are never created, because they'll never be used to build other digests.
///
/// Entries are pruned from the cache once digest block that is using this entry
/// is inserted (because digest block will includes all keys from this entry).
/// When there's a fork, entries are pruned when first changes trie is inserted.
pub struct BuildCache<H, N> {
/// Map of block (implies changes true) number => changes trie root.
roots_by_number: HashMap<N, H>,
/// Map of changes trie root => set of storage keys that are in this trie.
/// The `Option<Vec<u8>>` in inner `HashMap` stands for the child storage key.
/// If it is `None`, then the `HashSet` contains keys changed in top-level storage.
/// If it is `Some`, then the `HashSet` contains keys changed in child storage, identified by the key.
changed_keys: HashMap<H, HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>>,
}
/// The action to perform when block-with-changes-trie is imported.
#[derive(Debug, PartialEq)]
pub enum CacheAction<H, N> {
/// Cache data that has been collected when CT has been built.
CacheBuildData(CachedBuildData<H, N>),
/// Clear cache from all existing entries.
Clear,
}
/// The data that has been cached during changes trie building.
#[derive(Debug, PartialEq)]
pub struct CachedBuildData<H, N> {
block: N,
trie_root: H,
digest_input_blocks: Vec<N>,
changed_keys: HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>,
}
/// The action to perform when block-with-changes-trie is imported.
#[derive(Debug, PartialEq)]
pub(crate) enum IncompleteCacheAction<N> {
/// Cache data that has been collected when CT has been built.
CacheBuildData(IncompleteCachedBuildData<N>),
/// Clear cache from all existing entries.
Clear,
}
/// The data (without changes trie root) that has been cached during changes trie building.
#[derive(Debug, PartialEq)]
pub(crate) struct IncompleteCachedBuildData<N> {
digest_input_blocks: Vec<N>,
changed_keys: HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>,
}
impl<H, N> BuildCache<H, N>
where
N: Eq + ::std::hash::Hash,
H: Eq + ::std::hash::Hash + Clone,
{
/// Create new changes trie build cache.
pub fn new() -> Self {
BuildCache {
roots_by_number: HashMap::new(),
changed_keys: HashMap::new(),
}
}
/// Get cached changed keys for changes trie with given root.
pub fn get(&self, root: &H) -> Option<&HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>> {
self.changed_keys.get(&root)
}
/// Execute given functor with cached entry for given block.
/// Returns true if the functor has been called and false otherwise.
pub fn with_changed_keys(
&self,
root: &H,
functor: &mut dyn FnMut(&HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>),
) -> bool {
match self.changed_keys.get(&root) {
Some(changed_keys) => {
functor(changed_keys);
true
},
None => false,
}
}
/// Insert data into cache.
pub fn perform(&mut self, action: CacheAction<H, N>) {
match action {
CacheAction::CacheBuildData(data) => {
self.roots_by_number.insert(data.block, data.trie_root.clone());
self.changed_keys.insert(data.trie_root, data.changed_keys);
for digest_input_block in data.digest_input_blocks {
let digest_input_block_hash = self.roots_by_number.remove(&digest_input_block);
if let Some(digest_input_block_hash) = digest_input_block_hash {
self.changed_keys.remove(&digest_input_block_hash);
}
}
},
CacheAction::Clear => {
self.roots_by_number.clear();
self.changed_keys.clear();
},
}
}
}
impl<N> IncompleteCacheAction<N> {
/// Returns true if we need to collect changed keys for this action.
pub fn collects_changed_keys(&self) -> bool {
match *self {
IncompleteCacheAction::CacheBuildData(_) => true,
IncompleteCacheAction::Clear => false,
}
}
/// Complete cache action with computed changes trie root.
pub(crate) fn complete<H: Clone>(self, block: N, trie_root: &H) -> CacheAction<H, N> {
match self {
IncompleteCacheAction::CacheBuildData(build_data) =>
CacheAction::CacheBuildData(build_data.complete(block, trie_root.clone())),
IncompleteCacheAction::Clear => CacheAction::Clear,
}
}
/// Set numbers of blocks that are superseded by this new entry.
///
/// If/when this build data is committed to the cache, entries for these blocks
/// will be removed from the cache.
pub(crate) fn set_digest_input_blocks(self, digest_input_blocks: Vec<N>) -> Self {
match self {
IncompleteCacheAction::CacheBuildData(build_data) =>
IncompleteCacheAction::CacheBuildData(build_data.set_digest_input_blocks(digest_input_blocks)),
IncompleteCacheAction::Clear => IncompleteCacheAction::Clear,
}
}
/// Insert changed keys of given storage into cached data.
pub(crate) fn insert(
self,
storage_key: Option<Vec<u8>>,
changed_keys: HashSet<Vec<u8>>,
) -> Self {
match self {
IncompleteCacheAction::CacheBuildData(build_data) =>
IncompleteCacheAction::CacheBuildData(build_data.insert(storage_key, changed_keys)),
IncompleteCacheAction::Clear => IncompleteCacheAction::Clear,
}
}
}
impl<N> IncompleteCachedBuildData<N> {
/// Create new cached data.
pub(crate) fn new() -> Self {
IncompleteCachedBuildData {
digest_input_blocks: Vec::new(),
changed_keys: HashMap::new(),
}
}
fn complete<H>(self, block: N, trie_root: H) -> CachedBuildData<H, N> {
CachedBuildData {
block,
trie_root,
digest_input_blocks: self.digest_input_blocks,
changed_keys: self.changed_keys,
}
}
fn set_digest_input_blocks(mut self, digest_input_blocks: Vec<N>) -> Self {
self.digest_input_blocks = digest_input_blocks;
self
}
fn insert(
mut self,
storage_key: Option<Vec<u8>>,
changed_keys: HashSet<Vec<u8>>,
) -> Self {
self.changed_keys.insert(storage_key, changed_keys);
self
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn updated_keys_are_stored_when_non_top_level_digest_is_built() {
let mut data = IncompleteCachedBuildData::<u32>::new();
data = data.insert(None, vec![vec![1]].into_iter().collect());
assert_eq!(data.changed_keys.len(), 1);
let mut cache = BuildCache::new();
cache.perform(CacheAction::CacheBuildData(data.complete(1, 1)));
assert_eq!(cache.changed_keys.len(), 1);
assert_eq!(
cache.get(&1).unwrap().clone(),
vec![(None, vec![vec![1]].into_iter().collect())].into_iter().collect(),
);
}
#[test]
fn obsolete_entries_are_purged_when_new_ct_is_built() {
let mut cache = BuildCache::<u32, u32>::new();
cache.perform(CacheAction::CacheBuildData(IncompleteCachedBuildData::new()
.insert(None, vec![vec![1]].into_iter().collect())
.complete(1, 1)));
cache.perform(CacheAction::CacheBuildData(IncompleteCachedBuildData::new()
.insert(None, vec![vec![2]].into_iter().collect())
.complete(2, 2)));
cache.perform(CacheAction::CacheBuildData(IncompleteCachedBuildData::new()
.insert(None, vec![vec![3]].into_iter().collect())
.complete(3, 3)));
assert_eq!(cache.changed_keys.len(), 3);
cache.perform(CacheAction::CacheBuildData(IncompleteCachedBuildData::new()
.set_digest_input_blocks(vec![1, 2, 3])
.complete(4, 4)));
assert_eq!(cache.changed_keys.len(), 1);
cache.perform(CacheAction::CacheBuildData(IncompleteCachedBuildData::new()
.insert(None, vec![vec![8]].into_iter().collect())
.complete(8, 8)));
cache.perform(CacheAction::CacheBuildData(IncompleteCachedBuildData::new()
.insert(None, vec![vec![12]].into_iter().collect())
.complete(12, 12)));
assert_eq!(cache.changed_keys.len(), 3);
cache.perform(CacheAction::Clear);
assert_eq!(cache.changed_keys.len(), 0);
}
}
substrate/primitives/state-machine/src/changes_trie/build_iterator.rs
+431
View File
@@ -0,0 +1,431 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Structures and functions to return blocks whose changes are to be included
//! in given block's changes trie.
use num_traits::Zero;
use crate::changes_trie::{ConfigurationRange, BlockNumber};
/// Returns iterator of OTHER blocks that are required for inclusion into
/// changes trie of given block. Blocks are guaranteed to be returned in
/// ascending order.
///
/// Skewed digest is built IF block >= config.end.
pub fn digest_build_iterator<'a, Number: BlockNumber>(
config: ConfigurationRange<'a, Number>,
block: Number,
) -> DigestBuildIterator<Number> {
// prepare digest build parameters
let (_, _, digest_step) = match config.config.digest_level_at_block(config.zero, block.clone()) {
Some((current_level, digest_interval, digest_step)) =>
(current_level, digest_interval, digest_step),
None => return DigestBuildIterator::empty(),
};
DigestBuildIterator::new(block.clone(), config.end.unwrap_or(block), config.config.digest_interval, digest_step)
}
/// Changes trie build iterator that returns numbers of OTHER blocks that are
/// required for inclusion into changes trie of given block.
#[derive(Debug)]
pub struct DigestBuildIterator<Number: BlockNumber> {
/// Block we're building changes trie for. It could (logically) be a post-end block if we are creating
/// skewed digest.
block: Number,
/// Block that is a last block where current configuration is active. We have never yet created anything
/// after this block => digest that we're creating can't reference any blocks that are >= end.
end: Number,
/// Interval of L1 digest blocks.
digest_interval: u32,
/// Max step that could be used when digest is created.
max_step: u32,
// Mutable data below:
/// Step of current blocks range.
current_step: u32,
/// Reverse step of current blocks range.
current_step_reverse: u32,
/// Current blocks range.
current_range: Option<BlocksRange<Number>>,
/// Last block that we have returned.
last_block: Option<Number>,
}
impl<Number: BlockNumber> DigestBuildIterator<Number> {
/// Create new digest build iterator.
pub fn new(block: Number, end: Number, digest_interval: u32, max_step: u32) -> Self {
DigestBuildIterator {
block,
end,
digest_interval,
max_step,
current_step: max_step,
current_step_reverse: 0,
current_range: None,
last_block: None,
}
}
/// Create empty digest build iterator.
pub fn empty() -> Self {
Self::new(Zero::zero(), Zero::zero(), 0, 0)
}
}
impl<Number: BlockNumber> Iterator for DigestBuildIterator<Number> {
type Item = Number;
fn next(&mut self) -> Option<Self::Item> {
// when we're building skewed digest, we might want to skip some blocks if
// they're not covered by current configuration
loop {
if let Some(next) = self.current_range.as_mut().and_then(|iter| iter.next()) {
if next < self.end {
self.last_block = Some(next.clone());
return Some(next);
}
}
// we are safe to use non-checking mul/sub versions here because:
// DigestBuildIterator is created only by internal function that is checking
// that all multiplications/subtractions are safe within max_step limit
let next_step_reverse = if self.current_step_reverse == 0 {
1
} else {
self.current_step_reverse * self.digest_interval
};
if next_step_reverse > self.max_step {
return None;
}
self.current_step_reverse = next_step_reverse;
self.current_range = Some(BlocksRange::new(
match self.last_block.clone() {
Some(last_block) => last_block + self.current_step.into(),
None => self.block.clone() - (self.current_step * self.digest_interval - self.current_step).into(),
},
self.block.clone(),
self.current_step.into(),
));
self.current_step = self.current_step / self.digest_interval;
if self.current_step == 0 {
self.current_step = 1;
}
}
}
}
/// Blocks range iterator with builtin step_by support.
#[derive(Debug)]
struct BlocksRange<Number: BlockNumber> {
current: Number,
end: Number,
step: Number,
}
impl<Number: BlockNumber> BlocksRange<Number> {
pub fn new(begin: Number, end: Number, step: Number) -> Self {
BlocksRange {
current: begin,
end,
step,
}
}
}
impl<Number: BlockNumber> Iterator for BlocksRange<Number> {
type Item = Number;
fn next(&mut self) -> Option<Self::Item> {
if self.current >= self.end {
return None;
}
let current = Some(self.current.clone());
self.current += self.step.clone();
current
}
}
#[cfg(test)]
mod tests {
use crate::changes_trie::Configuration;
use super::*;
fn digest_build_iterator(
digest_interval: u32,
digest_levels: u32,
zero: u64,
block: u64,
end: Option<u64>,
) -> DigestBuildIterator<u64> {
super::digest_build_iterator(
ConfigurationRange {
config: &Configuration {
digest_interval,
digest_levels,
},
zero,
end,
},
block,
)
}
fn digest_build_iterator_basic(
digest_interval: u32,
digest_levels: u32,
zero: u64,
block: u64,
) -> (u64, u32, u32) {
let iter = digest_build_iterator(digest_interval, digest_levels, zero, block, None);
(iter.block, iter.digest_interval, iter.max_step)
}
fn digest_build_iterator_blocks(
digest_interval: u32,
digest_levels: u32,
zero: u64,
block: u64,
end: Option<u64>,
) -> Vec<u64> {
digest_build_iterator(digest_interval, digest_levels, zero, block, end).collect()
}
#[test]
fn suggest_digest_inclusion_returns_empty_iterator() {
fn test_with_zero(zero: u64) {
let empty = (0, 0, 0);
assert_eq!(digest_build_iterator_basic(4, 16, zero, zero + 0), empty, "block is 0");
assert_eq!(digest_build_iterator_basic(0, 16, zero, zero + 64), empty, "digest_interval is 0");
assert_eq!(digest_build_iterator_basic(1, 16, zero, zero + 64), empty, "digest_interval is 1");
assert_eq!(digest_build_iterator_basic(4, 0, zero, zero + 64), empty, "digest_levels is 0");
assert_eq!(
digest_build_iterator_basic(4, 16, zero, zero + 1),
empty,
"digest is not required for this block",
);
assert_eq!(
digest_build_iterator_basic(4, 16, zero, zero + 2),
empty,
"digest is not required for this block",
);
assert_eq!(
digest_build_iterator_basic(4, 16, zero, zero + 15),
empty,
"digest is not required for this block",
);
assert_eq!(
digest_build_iterator_basic(4, 16, zero, zero + 17),
empty,
"digest is not required for this block",
);
assert_eq!(digest_build_iterator_basic(
::std::u32::MAX / 2 + 1,
16,
zero,
::std::u64::MAX,
), empty, "digest_interval * 2 is greater than u64::MAX");
}
test_with_zero(0);
test_with_zero(1);
test_with_zero(2);
test_with_zero(4);
test_with_zero(17);
}
#[test]
fn suggest_digest_inclusion_returns_level1_iterator() {
fn test_with_zero(zero: u64) {
assert_eq!(
digest_build_iterator_basic(16, 1, zero, zero + 16),
(zero + 16, 16, 1),
"!(block % interval) && first digest level == block",
);
assert_eq!(
digest_build_iterator_basic(16, 1, zero, zero + 256),
(zero + 256, 16, 1),
"!(block % interval^2), but there's only 1 digest level",
);
assert_eq!(
digest_build_iterator_basic(16, 2, zero, zero + 32),
(zero + 32, 16, 1),
"second level digest is not required for this block",
);
assert_eq!(
digest_build_iterator_basic(16, 3, zero, zero + 4080),
(zero + 4080, 16, 1),
"second && third level digest are not required for this block",
);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn suggest_digest_inclusion_returns_level2_iterator() {
fn test_with_zero(zero: u64) {
assert_eq!(
digest_build_iterator_basic(16, 2, zero, zero + 256),
(zero + 256, 16, 16),
"second level digest",
);
assert_eq!(
digest_build_iterator_basic(16, 2, zero, zero + 4096),
(zero + 4096, 16, 16),
"!(block % interval^3), but there's only 2 digest levels",
);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn suggest_digest_inclusion_returns_level3_iterator() {
fn test_with_zero(zero: u64) {
assert_eq!(
digest_build_iterator_basic(16, 3, zero, zero + 4096),
(zero + 4096, 16, 256),
"third level digest: beginning",
);
assert_eq!(
digest_build_iterator_basic(16, 3, zero, zero + 8192),
(zero + 8192, 16, 256),
"third level digest: next",
);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn digest_iterator_returns_level1_blocks() {
fn test_with_zero(zero: u64) {
assert_eq!(digest_build_iterator_blocks(16, 1, zero, zero + 16, None),
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
.iter().map(|item| zero + item).collect::<Vec<_>>());
assert_eq!(digest_build_iterator_blocks(16, 1, zero, zero + 256, None),
[241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255]
.iter().map(|item| zero + item).collect::<Vec<_>>());
assert_eq!(digest_build_iterator_blocks(16, 2, zero, zero + 32, None),
[17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
.iter().map(|item| zero + item).collect::<Vec<_>>());
assert_eq!(digest_build_iterator_blocks(16, 3, zero, zero + 4080, None),
[4065, 4066, 4067, 4068, 4069, 4070, 4071, 4072, 4073, 4074, 4075, 4076, 4077, 4078, 4079]
.iter().map(|item| zero + item).collect::<Vec<_>>());
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn digest_iterator_returns_level1_and_level2_blocks() {
fn test_with_zero(zero: u64) {
assert_eq!(digest_build_iterator_blocks(16, 2, zero, zero + 256, None),
[
// level2 points to previous 16-1 level1 digests:
16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240,
// level2 is a level1 digest of 16-1 previous blocks:
241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255,
].iter().map(|item| zero + item).collect::<Vec<_>>(),
);
assert_eq!(digest_build_iterator_blocks(16, 2, zero, zero + 4096, None),
[
// level2 points to previous 16-1 level1 digests:
3856, 3872, 3888, 3904, 3920, 3936, 3952, 3968, 3984, 4000, 4016, 4032, 4048, 4064, 4080,
// level2 is a level1 digest of 16-1 previous blocks:
4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093, 4094, 4095,
].iter().map(|item| zero + item).collect::<Vec<_>>(),
);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn digest_iterator_returns_level1_and_level2_and_level3_blocks() {
fn test_with_zero(zero: u64) {
assert_eq!(digest_build_iterator_blocks(16, 3, zero, zero + 4096, None),
[
// level3 points to previous 16-1 level2 digests:
256, 512, 768, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840,
// level3 points to previous 16-1 level1 digests:
3856, 3872, 3888, 3904, 3920, 3936, 3952, 3968, 3984, 4000, 4016, 4032, 4048, 4064, 4080,
// level3 is a level1 digest of 16-1 previous blocks:
4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093, 4094, 4095,
].iter().map(|item| zero + item).collect::<Vec<_>>(),
);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn digest_iterator_returns_skewed_digest_blocks() {
fn test_with_zero(zero: u64) {
assert_eq!(digest_build_iterator_blocks(16, 3, zero, zero + 4096, Some(zero + 1338)),
[
// level3 MUST point to previous 16-1 level2 digests, BUT there are only 5:
256, 512, 768, 1024, 1280,
// level3 MUST point to previous 16-1 level1 digests, BUT there are only 3:
1296, 1312, 1328,
// level3 MUST be a level1 digest of 16-1 previous blocks, BUT there are only 9:
1329, 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337,
].iter().map(|item| zero + item).collect::<Vec<_>>(),
);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
#[test]
fn digest_iterator_returns_skewed_digest_blocks_skipping_level() {
fn test_with_zero(zero: u64) {
assert_eq!(digest_build_iterator_blocks(16, 3, zero, zero + 4096, Some(zero + 1284)),
[
// level3 MUST point to previous 16-1 level2 digests, BUT there are only 5:
256, 512, 768, 1024, 1280,
// level3 MUST point to previous 16-1 level1 digests, BUT there are NO ANY L1-digests:
// level3 MUST be a level1 digest of 16-1 previous blocks, BUT there are only 3:
1281, 1282, 1283,
].iter().map(|item| zero + item).collect::<Vec<_>>(),
);
}
test_with_zero(0);
test_with_zero(16);
test_with_zero(17);
}
}
substrate/primitives/state-machine/src/changes_trie/changes_iterator.rs
+635
View File
@@ -0,0 +1,635 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Functions + iterator that traverses changes tries and returns all
//! (block, extrinsic) pairs where given key has been changed.
use std::cell::RefCell;
use std::collections::VecDeque;
use codec::{Decode, Encode};
use hash_db::Hasher;
use num_traits::Zero;
use trie::Recorder;
use crate::changes_trie::{AnchorBlockId, ConfigurationRange, RootsStorage, Storage, BlockNumber};
use crate::changes_trie::input::{DigestIndex, ExtrinsicIndex, DigestIndexValue, ExtrinsicIndexValue};
use crate::changes_trie::storage::{TrieBackendAdapter, InMemoryStorage};
use crate::changes_trie::input::ChildIndex;
use crate::changes_trie::surface_iterator::{surface_iterator, SurfaceIterator};
use crate::proving_backend::ProvingBackendRecorder;
use crate::trie_backend_essence::{TrieBackendEssence};
/// Return changes of given key at given blocks range.
/// `max` is the number of best known block.
/// Changes are returned in descending order (i.e. last block comes first).
pub fn key_changes<'a, H: Hasher, Number: BlockNumber>(
config: ConfigurationRange<'a, Number>,
storage: &'a dyn Storage<H, Number>,
begin: Number,
end: &'a AnchorBlockId<H::Out, Number>,
max: Number,
storage_key: Option<&'a [u8]>,
key: &'a [u8],
) -> Result<DrilldownIterator<'a, H, Number>, String> {
// we can't query any roots before root
let max = ::std::cmp::min(max.clone(), end.number.clone());
Ok(DrilldownIterator {
essence: DrilldownIteratorEssence {
storage_key,
key,
roots_storage: storage.as_roots_storage(),
storage,
begin: begin.clone(),
end,
config: config.clone(),
surface: surface_iterator(
config,
max,
begin,
end.number.clone(),
)?,
extrinsics: Default::default(),
blocks: Default::default(),
_hasher: ::std::marker::PhantomData::<H>::default(),
},
})
}
/// Returns proof of changes of given key at given blocks range.
/// `max` is the number of best known block.
pub fn key_changes_proof<'a, H: Hasher, Number: BlockNumber>(
config: ConfigurationRange<'a, Number>,
storage: &dyn Storage<H, Number>,
begin: Number,
end: &AnchorBlockId<H::Out, Number>,
max: Number,
storage_key: Option<&[u8]>,
key: &[u8],
) -> Result<Vec<Vec<u8>>, String> {
// we can't query any roots before root
let max = ::std::cmp::min(max.clone(), end.number.clone());
let mut iter = ProvingDrilldownIterator {
essence: DrilldownIteratorEssence {
storage_key,
key,
roots_storage: storage.as_roots_storage(),
storage,
begin: begin.clone(),
end,
config: config.clone(),
surface: surface_iterator(
config,
max,
begin,
end.number.clone(),
)?,
extrinsics: Default::default(),
blocks: Default::default(),
_hasher: ::std::marker::PhantomData::<H>::default(),
},
proof_recorder: Default::default(),
};
// iterate to collect proof
while let Some(item) = iter.next() {
item?;
}
Ok(iter.extract_proof())
}
/// Check key changes proof and return changes of the key at given blocks range.
/// `max` is the number of best known block.
/// Changes are returned in descending order (i.e. last block comes first).
pub fn key_changes_proof_check<'a, H: Hasher, Number: BlockNumber>(
config: ConfigurationRange<'a, Number>,
roots_storage: &dyn RootsStorage<H, Number>,
proof: Vec<Vec<u8>>,
begin: Number,
end: &AnchorBlockId<H::Out, Number>,
max: Number,
storage_key: Option<&[u8]>,
key: &[u8]
) -> Result<Vec<(Number, u32)>, String> {
key_changes_proof_check_with_db(
config,
roots_storage,
&InMemoryStorage::with_proof(proof),
begin,
end,
max,
storage_key,
key,
)
}
/// Similar to the `key_changes_proof_check` function, but works with prepared proof storage.
pub fn key_changes_proof_check_with_db<'a, H: Hasher, Number: BlockNumber>(
config: ConfigurationRange<'a, Number>,
roots_storage: &dyn RootsStorage<H, Number>,
proof_db: &InMemoryStorage<H, Number>,
begin: Number,
end: &AnchorBlockId<H::Out, Number>,
max: Number,
storage_key: Option<&[u8]>,
key: &[u8]
) -> Result<Vec<(Number, u32)>, String> {
// we can't query any roots before root
let max = ::std::cmp::min(max.clone(), end.number.clone());
DrilldownIterator {
essence: DrilldownIteratorEssence {
storage_key,
key,
roots_storage,
storage: proof_db,
begin: begin.clone(),
end,
config: config.clone(),
surface: surface_iterator(
config,
max,
begin,
end.number.clone(),
)?,
extrinsics: Default::default(),
blocks: Default::default(),
_hasher: ::std::marker::PhantomData::<H>::default(),
},
}.collect()
}
/// Drilldown iterator - receives 'digest points' from surface iterator and explores
/// every point until extrinsic is found.
pub struct DrilldownIteratorEssence<'a, H, Number>
where
H: Hasher,
Number: BlockNumber,
H::Out: 'a,
{
storage_key: Option<&'a [u8]>,
key: &'a [u8],
roots_storage: &'a dyn RootsStorage<H, Number>,
storage: &'a dyn Storage<H, Number>,
begin: Number,
end: &'a AnchorBlockId<H::Out, Number>,
config: ConfigurationRange<'a, Number>,
surface: SurfaceIterator<'a, Number>,
extrinsics: VecDeque<(Number, u32)>,
blocks: VecDeque<(Number, Option<u32>)>,
_hasher: ::std::marker::PhantomData<H>,
}
impl<'a, H, Number> DrilldownIteratorEssence<'a, H, Number>
where
H: Hasher,
Number: BlockNumber,
H::Out: 'a,
{
pub fn next<F>(&mut self, trie_reader: F) -> Option<Result<(Number, u32), String>>
where
F: FnMut(&dyn Storage<H, Number>, H::Out, &[u8]) -> Result<Option<Vec<u8>>, String>,
{
match self.do_next(trie_reader) {
Ok(Some(res)) => Some(Ok(res)),
Ok(None) => None,
Err(err) => Some(Err(err)),
}
}
fn do_next<F>(&mut self, mut trie_reader: F) -> Result<Option<(Number, u32)>, String>
where
F: FnMut(&dyn Storage<H, Number>, H::Out, &[u8]) -> Result<Option<Vec<u8>>, String>,
{
loop {
if let Some((block, extrinsic)) = self.extrinsics.pop_front() {
return Ok(Some((block, extrinsic)));
}
if let Some((block, level)) = self.blocks.pop_front() {
// not having a changes trie root is an error because:
// we never query roots for future blocks
// AND trie roots for old blocks are known (both on full + light node)
let trie_root = self.roots_storage.root(&self.end, block.clone())?
.ok_or_else(|| format!("Changes trie root for block {} is not found", block.clone()))?;
let trie_root = if let Some(storage_key) = self.storage_key {
let child_key = ChildIndex {
block: block.clone(),
storage_key: storage_key.to_vec(),
}.encode();
if let Some(trie_root) = trie_reader(self.storage, trie_root, &child_key)?
.and_then(|v| <Vec<u8>>::decode(&mut &v[..]).ok())
.map(|v| {
let mut hash = H::Out::default();
hash.as_mut().copy_from_slice(&v[..]);
hash
}) {
trie_root
} else {
continue;
}
} else {
trie_root
};
// only return extrinsics for blocks before self.max
// most of blocks will be filtered out before pushing to `self.blocks`
// here we just throwing away changes at digest blocks we're processing
debug_assert!(block >= self.begin, "We shall not touch digests earlier than a range' begin");
if block <= self.end.number {
let extrinsics_key = ExtrinsicIndex { block: block.clone(), key: self.key.to_vec() }.encode();
let extrinsics = trie_reader(self.storage, trie_root, &extrinsics_key);
if let Some(extrinsics) = extrinsics? {
if let Ok(extrinsics) = ExtrinsicIndexValue::decode(&mut &extrinsics[..]) {
self.extrinsics.extend(extrinsics.into_iter().rev().map(|e| (block.clone(), e)));
}
}
}
let blocks_key = DigestIndex { block: block.clone(), key: self.key.to_vec() }.encode();
let blocks = trie_reader(self.storage, trie_root, &blocks_key);
if let Some(blocks) = blocks? {
if let Ok(blocks) = <DigestIndexValue<Number>>::decode(&mut &blocks[..]) {
// filter level0 blocks here because we tend to use digest blocks,
// AND digest block changes could also include changes for out-of-range blocks
let begin = self.begin.clone();
let end = self.end.number.clone();
let config = self.config.clone();
self.blocks.extend(blocks.into_iter()
.rev()
.filter(|b| level.map(|level| level > 1).unwrap_or(true) || (*b >= begin && *b <= end))
.map(|b| {
let prev_level = level
.map(|level| Some(level - 1))
.unwrap_or_else(||
Some(config.config.digest_level_at_block(config.zero.clone(), b.clone())
.map(|(level, _, _)| level)
.unwrap_or_else(|| Zero::zero())));
(b, prev_level)
})
);
}
}
continue;
}
match self.surface.next() {
Some(Ok(block)) => self.blocks.push_back(block),
Some(Err(err)) => return Err(err),
None => return Ok(None),
}
}
}
}
/// Exploring drilldown operator.
pub struct DrilldownIterator<'a, H, Number>
where
Number: BlockNumber,
H: Hasher,
H::Out: 'a,
{
essence: DrilldownIteratorEssence<'a, H, Number>,
}
impl<'a, H: Hasher, Number: BlockNumber> Iterator
for DrilldownIterator<'a, H, Number>
{
type Item = Result<(Number, u32), String>;
fn next(&mut self) -> Option<Self::Item> {
self.essence.next(|storage, root, key|
TrieBackendEssence::<_, H>::new(TrieBackendAdapter::new(storage), root).storage(key))
}
}
/// Proving drilldown iterator.
struct ProvingDrilldownIterator<'a, H, Number>
where
Number: BlockNumber,
H: Hasher,
H::Out: 'a,
{
essence: DrilldownIteratorEssence<'a, H, Number>,
proof_recorder: RefCell<Recorder<H::Out>>,
}
impl<'a, H, Number> ProvingDrilldownIterator<'a, H, Number>
where
Number: BlockNumber,
H: Hasher,
H::Out: 'a,
{
/// Consume the iterator, extracting the gathered proof in lexicographical order
/// by value.
pub fn extract_proof(self) -> Vec<Vec<u8>> {
self.proof_recorder.into_inner().drain()
.into_iter()
.map(|n| n.data.to_vec())
.collect()
}
}
impl<'a, H, Number> Iterator for ProvingDrilldownIterator<'a, H, Number>
where
Number: BlockNumber,
H: Hasher,
H::Out: 'a,
{
type Item = Result<(Number, u32), String>;
fn next(&mut self) -> Option<Self::Item> {
let proof_recorder = &mut *self.proof_recorder.try_borrow_mut()
.expect("only fails when already borrowed; storage() is non-reentrant; qed");
self.essence.next(|storage, root, key|
ProvingBackendRecorder::<_, H> {
backend: &TrieBackendEssence::new(TrieBackendAdapter::new(storage), root),
proof_recorder,
}.storage(key))
}
}
#[cfg(test)]
mod tests {
use std::iter::FromIterator;
use primitives::Blake2Hasher;
use crate::changes_trie::Configuration;
use crate::changes_trie::input::InputPair;
use crate::changes_trie::storage::InMemoryStorage;
use super::*;
fn prepare_for_drilldown() -> (Configuration, InMemoryStorage<Blake2Hasher, u64>) {
let config = Configuration { digest_interval: 4, digest_levels: 2 };
let backend = InMemoryStorage::with_inputs(vec![
// digest: 1..4 => [(3, 0)]
(1, vec![
]),
(2, vec![
]),
(3, vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 3, key: vec![42] }, vec![0]),
]),
(4, vec![
InputPair::DigestIndex(DigestIndex { block: 4, key: vec![42] }, vec![3]),
]),
// digest: 5..8 => [(6, 3), (8, 1+2)]
(5, vec![]),
(6, vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 6, key: vec![42] }, vec![3]),
]),
(7, vec![]),
(8, vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 8, key: vec![42] }, vec![1, 2]),
InputPair::DigestIndex(DigestIndex { block: 8, key: vec![42] }, vec![6]),
]),
// digest: 9..12 => []
(9, vec![]),
(10, vec![]),
(11, vec![]),
(12, vec![]),
// digest: 0..16 => [4, 8]
(13, vec![]),
(14, vec![]),
(15, vec![]),
(16, vec![
InputPair::DigestIndex(DigestIndex { block: 16, key: vec![42] }, vec![4, 8]),
]),
], vec![(b"1".to_vec(), vec![
(1, vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 1, key: vec![42] }, vec![0]),
]),
(2, vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 2, key: vec![42] }, vec![3]),
]),
(16, vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![42] }, vec![5]),
InputPair::DigestIndex(DigestIndex { block: 16, key: vec![42] }, vec![2]),
]),
]),
]);
(config, backend)
}
fn configuration_range<'a>(config: &'a Configuration, zero: u64) -> ConfigurationRange<'a, u64> {
ConfigurationRange {
config,
zero,
end: None,
}
}
#[test]
fn drilldown_iterator_works() {
let (config, storage) = prepare_for_drilldown();
let drilldown_result = key_changes::<Blake2Hasher, u64>(
configuration_range(&config, 0),
&storage,
1,
&AnchorBlockId { hash: Default::default(), number: 16 },
16,
None,
&[42],
).and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)]));
let drilldown_result = key_changes::<Blake2Hasher, u64>(
configuration_range(&config, 0),
&storage,
1,
&AnchorBlockId { hash: Default::default(), number: 2 },
4,
None,
&[42],
).and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![]));
let drilldown_result = key_changes::<Blake2Hasher, u64>(
configuration_range(&config, 0),
&storage,
1,
&AnchorBlockId { hash: Default::default(), number: 3 },
4,
None,
&[42],
).and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(3, 0)]));
let drilldown_result = key_changes::<Blake2Hasher, u64>(
configuration_range(&config, 0),
&storage,
1,
&AnchorBlockId { hash: Default::default(), number: 7 },
7,
None,
&[42],
).and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(6, 3), (3, 0)]));
let drilldown_result = key_changes::<Blake2Hasher, u64>(
configuration_range(&config, 0),
&storage,
7,
&AnchorBlockId { hash: Default::default(), number: 8 },
8,
None,
&[42],
).and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1)]));
let drilldown_result = key_changes::<Blake2Hasher, u64>(
configuration_range(&config, 0),
&storage,
5,
&AnchorBlockId { hash: Default::default(), number: 7 },
8,
None,
&[42],
).and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(6, 3)]));
}
#[test]
fn drilldown_iterator_fails_when_storage_fails() {
let (config, storage) = prepare_for_drilldown();
storage.clear_storage();
assert!(key_changes::<Blake2Hasher, u64>(
configuration_range(&config, 0),
&storage,
1,
&AnchorBlockId { hash: Default::default(), number: 100 },
1000,
None,
&[42],
).and_then(|i| i.collect::<Result<Vec<_>, _>>()).is_err());
assert!(key_changes::<Blake2Hasher, u64>(
configuration_range(&config, 0),
&storage,
1,
&AnchorBlockId { hash: Default::default(), number: 100 },
1000,
Some(&b"1"[..]),
&[42],
).and_then(|i| i.collect::<Result<Vec<_>, _>>()).is_err());
}
#[test]
fn drilldown_iterator_fails_when_range_is_invalid() {
let (config, storage) = prepare_for_drilldown();
assert!(key_changes::<Blake2Hasher, u64>(
configuration_range(&config, 0),
&storage,
1,
&AnchorBlockId { hash: Default::default(), number: 100 },
50,
None,
&[42],
).is_err());
assert!(key_changes::<Blake2Hasher, u64>(
configuration_range(&config, 0),
&storage,
20,
&AnchorBlockId { hash: Default::default(), number: 10 },
100,
None,
&[42],
).is_err());
}
#[test]
fn proving_drilldown_iterator_works() {
// happens on remote full node:
// create drilldown iterator that records all trie nodes during drilldown
let (remote_config, remote_storage) = prepare_for_drilldown();
let remote_proof = key_changes_proof::<Blake2Hasher, u64>(
configuration_range(&remote_config, 0), &remote_storage, 1,
&AnchorBlockId { hash: Default::default(), number: 16 }, 16, None, &[42]).unwrap();
let (remote_config, remote_storage) = prepare_for_drilldown();
let remote_proof_child = key_changes_proof::<Blake2Hasher, u64>(
configuration_range(&remote_config, 0), &remote_storage, 1,
&AnchorBlockId { hash: Default::default(), number: 16 }, 16, Some(&b"1"[..]), &[42]).unwrap();
// happens on local light node:
// create drilldown iterator that works the same, but only depends on trie
let (local_config, local_storage) = prepare_for_drilldown();
local_storage.clear_storage();
let local_result = key_changes_proof_check::<Blake2Hasher, u64>(
configuration_range(&local_config, 0), &local_storage, remote_proof, 1,
&AnchorBlockId { hash: Default::default(), number: 16 }, 16, None, &[42]);
let (local_config, local_storage) = prepare_for_drilldown();
local_storage.clear_storage();
let local_result_child = key_changes_proof_check::<Blake2Hasher, u64>(
configuration_range(&local_config, 0), &local_storage, remote_proof_child, 1,
&AnchorBlockId { hash: Default::default(), number: 16 }, 16, Some(&b"1"[..]), &[42]);
// check that drilldown result is the same as if it was happening at the full node
assert_eq!(local_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)]));
assert_eq!(local_result_child, Ok(vec![(16, 5), (2, 3)]));
}
#[test]
fn drilldown_iterator_works_with_skewed_digest() {
let config = Configuration { digest_interval: 4, digest_levels: 3 };
let mut config_range = configuration_range(&config, 0);
config_range.end = Some(91);
// when 4^3 deactivates at block 91:
// last L3 digest has been created at block#64
// skewed digest covers:
// L2 digests at blocks: 80
// L1 digests at blocks: 84, 88
// regular blocks: 89, 90, 91
let mut input = (1u64..92u64).map(|b| (b, vec![])).collect::<Vec<_>>();
// changed at block#63 and covered by L3 digest at block#64
input[63 - 1].1.push(InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 63, key: vec![42] }, vec![0]));
input[64 - 1].1.push(InputPair::DigestIndex(DigestIndex { block: 64, key: vec![42] }, vec![63]));
// changed at block#79 and covered by L2 digest at block#80 + skewed digest at block#91
input[79 - 1].1.push(InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 79, key: vec![42] }, vec![1]));
input[80 - 1].1.push(InputPair::DigestIndex(DigestIndex { block: 80, key: vec![42] }, vec![79]));
input[91 - 1].1.push(InputPair::DigestIndex(DigestIndex { block: 91, key: vec![42] }, vec![80]));
let storage = InMemoryStorage::with_inputs(input, vec![]);
let drilldown_result = key_changes::<Blake2Hasher, u64>(
config_range,
&storage,
1,
&AnchorBlockId { hash: Default::default(), number: 91 },
100_000u64,
None,
&[42],
).and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(79, 1), (63, 0)]));
}
}
substrate/primitives/state-machine/src/changes_trie/input.rs
+204
View File
@@ -0,0 +1,204 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Different types of changes trie input pairs.
use codec::{Decode, Encode, Input, Output, Error};
use crate::changes_trie::BlockNumber;
/// Key of { changed key => set of extrinsic indices } mapping.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ExtrinsicIndex<Number: BlockNumber> {
/// Block at which this key has been inserted in the trie.
pub block: Number,
/// Storage key this node is responsible for.
pub key: Vec<u8>,
}
/// Value of { changed key => set of extrinsic indices } mapping.
pub type ExtrinsicIndexValue = Vec<u32>;
/// Key of { changed key => block/digest block numbers } mapping.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DigestIndex<Number: BlockNumber> {
/// Block at which this key has been inserted in the trie.
pub block: Number,
/// Storage key this node is responsible for.
pub key: Vec<u8>,
}
/// Key of { childtrie key => Childchange trie } mapping.
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct ChildIndex<Number: BlockNumber> {
/// Block at which this key has been inserted in the trie.
pub block: Number,
/// Storage key this node is responsible for.
pub storage_key: Vec<u8>,
}
/// Value of { changed key => block/digest block numbers } mapping.
pub type DigestIndexValue<Number> = Vec<Number>;
/// Value of { changed key => block/digest block numbers } mapping.
/// That is the root of the child change trie.
pub type ChildIndexValue = Vec<u8>;
/// Single input pair of changes trie.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum InputPair<Number: BlockNumber> {
/// Element of { key => set of extrinsics where key has been changed } element mapping.
ExtrinsicIndex(ExtrinsicIndex<Number>, ExtrinsicIndexValue),
/// Element of { key => set of blocks/digest blocks where key has been changed } element mapping.
DigestIndex(DigestIndex<Number>, DigestIndexValue<Number>),
/// Element of { childtrie key => Childchange trie } where key has been changed } element mapping.
ChildIndex(ChildIndex<Number>, ChildIndexValue),
}
/// Single input key of changes trie.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum InputKey<Number: BlockNumber> {
/// Key of { key => set of extrinsics where key has been changed } element mapping.
ExtrinsicIndex(ExtrinsicIndex<Number>),
/// Key of { key => set of blocks/digest blocks where key has been changed } element mapping.
DigestIndex(DigestIndex<Number>),
/// Key of { childtrie key => Childchange trie } where key has been changed } element mapping.
ChildIndex(ChildIndex<Number>),
}
impl<Number: BlockNumber> InputPair<Number> {
/// Extract storage key that this pair corresponds to.
pub fn key(&self) -> Option<&[u8]> {
match *self {
InputPair::ExtrinsicIndex(ref key, _) => Some(&key.key),
InputPair::DigestIndex(ref key, _) => Some(&key.key),
InputPair::ChildIndex(_, _) => None,
}
}
}
impl<Number: BlockNumber> Into<(Vec<u8>, Vec<u8>)> for InputPair<Number> {
fn into(self) -> (Vec<u8>, Vec<u8>) {
match self {
InputPair::ExtrinsicIndex(key, value) => (key.encode(), value.encode()),
InputPair::DigestIndex(key, value) => (key.encode(), value.encode()),
InputPair::ChildIndex(key, value) => (key.encode(), value.encode()),
}
}
}
impl<Number: BlockNumber> Into<InputKey<Number>> for InputPair<Number> {
fn into(self) -> InputKey<Number> {
match self {
InputPair::ExtrinsicIndex(key, _) => InputKey::ExtrinsicIndex(key),
InputPair::DigestIndex(key, _) => InputKey::DigestIndex(key),
InputPair::ChildIndex(key, _) => InputKey::ChildIndex(key),
}
}
}
impl<Number: BlockNumber> ExtrinsicIndex<Number> {
pub fn key_neutral_prefix(block: Number) -> Vec<u8> {
let mut prefix = vec![1];
prefix.extend(block.encode());
prefix
}
}
impl<Number: BlockNumber> Encode for ExtrinsicIndex<Number> {
fn encode_to<W: Output>(&self, dest: &mut W) {
dest.push_byte(1);
self.block.encode_to(dest);
self.key.encode_to(dest);
}
}
impl<Number: BlockNumber> codec::EncodeLike for ExtrinsicIndex<Number> {}
impl<Number: BlockNumber> DigestIndex<Number> {
pub fn key_neutral_prefix(block: Number) -> Vec<u8> {
let mut prefix = vec![2];
prefix.extend(block.encode());
prefix
}
}
impl<Number: BlockNumber> Encode for DigestIndex<Number> {
fn encode_to<W: Output>(&self, dest: &mut W) {
dest.push_byte(2);
self.block.encode_to(dest);
self.key.encode_to(dest);
}
}
impl<Number: BlockNumber> ChildIndex<Number> {
pub fn key_neutral_prefix(block: Number) -> Vec<u8> {
let mut prefix = vec![3];
prefix.extend(block.encode());
prefix
}
}
impl<Number: BlockNumber> Encode for ChildIndex<Number> {
fn encode_to<W: Output>(&self, dest: &mut W) {
dest.push_byte(3);
self.block.encode_to(dest);
self.storage_key.encode_to(dest);
}
}
impl<Number: BlockNumber> codec::EncodeLike for DigestIndex<Number> {}
impl<Number: BlockNumber> Decode for InputKey<Number> {
fn decode<I: Input>(input: &mut I) -> Result<Self, Error> {
match input.read_byte()? {
1 => Ok(InputKey::ExtrinsicIndex(ExtrinsicIndex {
block: Decode::decode(input)?,
key: Decode::decode(input)?,
})),
2 => Ok(InputKey::DigestIndex(DigestIndex {
block: Decode::decode(input)?,
key: Decode::decode(input)?,
})),
3 => Ok(InputKey::ChildIndex(ChildIndex {
block: Decode::decode(input)?,
storage_key: Decode::decode(input)?,
})),
_ => Err("Invalid input key variant".into()),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn extrinsic_index_serialized_and_deserialized() {
let original = ExtrinsicIndex { block: 777u64, key: vec![42] };
let serialized = original.encode();
let deserialized: InputKey<u64> = Decode::decode(&mut &serialized[..]).unwrap();
assert_eq!(InputKey::ExtrinsicIndex(original), deserialized);
}
#[test]
fn digest_index_serialized_and_deserialized() {
let original = DigestIndex { block: 777u64, key: vec![42] };
let serialized = original.encode();
let deserialized: InputKey<u64> = Decode::decode(&mut &serialized[..]).unwrap();
assert_eq!(InputKey::DigestIndex(original), deserialized);
}
}
substrate/primitives/state-machine/src/changes_trie/mod.rs
+336
View File
@@ -0,0 +1,336 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Changes trie related structures and functions.
//!
//! Changes trie is a trie built of { storage key => extrinsiscs } pairs
//! at the end of each block. For every changed storage key it contains
//! a pair, mapping key to the set of extrinsics where it has been changed.
//!
//! Optionally, every N blocks, additional level1-digest nodes are appended
//! to the changes trie, containing pairs { storage key => blocks }. For every
//! storage key that has been changed in PREVIOUS N-1 blocks (except for genesis
//! block) it contains a pair, mapping this key to the set of blocks where it
//! has been changed.
//!
//! Optionally, every N^digest_level (where digest_level > 1) blocks, additional
//! digest_level digest is created. It is built out of pairs { storage key => digest
//! block }, containing entries for every storage key that has been changed in
//! the last N*digest_level-1 blocks (except for genesis block), mapping these keys
//! to the set of lower-level digest blocks.
//!
//! Changes trie configuration could change within a time. The range of blocks, where
//! configuration has been active, is given by two blocks: zero and end. Zero block is
//! the block where configuration has been set. But the first changes trie that uses
//! this configuration will be built at the block zero+1. If configuration deactivates
//! at some block, this will be the end block of the configuration. It is also the
//! zero block of the next configuration.
//!
//! If configuration has the end block, it also means that 'skewed digest' has/should
//! been built at that block. If this is the block where max-level digest should have
//! been created, than it is simply max-level digest of this configuration. Otherwise,
//! it is the digest that covers all blocks since last max-level digest block was
//! created.
//!
//! Changes trie only contains the top level storage changes. Sub-level changes
//! are propagated through its storage root on the top level storage.
mod build;
mod build_cache;
mod build_iterator;
mod changes_iterator;
mod input;
mod prune;
mod storage;
mod surface_iterator;
pub use self::build_cache::{BuildCache, CachedBuildData, CacheAction};
pub use self::storage::InMemoryStorage;
pub use self::changes_iterator::{
key_changes, key_changes_proof,
key_changes_proof_check, key_changes_proof_check_with_db,
};
pub use self::prune::{prune, oldest_non_pruned_trie};
use std::collections::{HashMap, HashSet};
use std::convert::TryInto;
use hash_db::{Hasher, Prefix};
use crate::backend::Backend;
use num_traits::{One, Zero};
use codec::{Decode, Encode};
use primitives;
use crate::changes_trie::build::prepare_input;
use crate::changes_trie::build_cache::{IncompleteCachedBuildData, IncompleteCacheAction};
use crate::overlayed_changes::OverlayedChanges;
use trie::{MemoryDB, DBValue, TrieMut};
use trie::trie_types::TrieDBMut;
/// Changes that are made outside of extrinsics are marked with this index;
pub const NO_EXTRINSIC_INDEX: u32 = 0xffffffff;
/// Requirements for block number that can be used with changes tries.
pub trait BlockNumber:
Send + Sync + 'static +
::std::fmt::Display +
Clone +
From<u32> + TryInto<u32> + One + Zero +
PartialEq + Ord +
::std::hash::Hash +
::std::ops::Add<Self, Output=Self> + ::std::ops::Sub<Self, Output=Self> +
::std::ops::Mul<Self, Output=Self> + ::std::ops::Div<Self, Output=Self> +
::std::ops::Rem<Self, Output=Self> +
::std::ops::AddAssign<Self> +
num_traits::CheckedMul + num_traits::CheckedSub +
Decode + Encode
{}
impl<T> BlockNumber for T where T:
Send + Sync + 'static +
::std::fmt::Display +
Clone +
From<u32> + TryInto<u32> + One + Zero +
PartialEq + Ord +
::std::hash::Hash +
::std::ops::Add<Self, Output=Self> + ::std::ops::Sub<Self, Output=Self> +
::std::ops::Mul<Self, Output=Self> + ::std::ops::Div<Self, Output=Self> +
::std::ops::Rem<Self, Output=Self> +
::std::ops::AddAssign<Self> +
num_traits::CheckedMul + num_traits::CheckedSub +
Decode + Encode,
{}
/// Block identifier that could be used to determine fork of this block.
#[derive(Debug)]
pub struct AnchorBlockId<Hash: ::std::fmt::Debug, Number: BlockNumber> {
/// Hash of this block.
pub hash: Hash,
/// Number of this block.
pub number: Number,
}
/// Changes trie storage. Provides access to trie roots and trie nodes.
pub trait RootsStorage<H: Hasher, Number: BlockNumber>: Send + Sync {
/// Resolve hash of the block into anchor.
fn build_anchor(&self, hash: H::Out) -> Result<AnchorBlockId<H::Out, Number>, String>;
/// Get changes trie root for the block with given number which is an ancestor (or the block
/// itself) of the anchor_block (i.e. anchor_block.number >= block).
fn root(&self, anchor: &AnchorBlockId<H::Out, Number>, block: Number) -> Result<Option<H::Out>, String>;
}
/// Changes trie storage. Provides access to trie roots and trie nodes.
pub trait Storage<H: Hasher, Number: BlockNumber>: RootsStorage<H, Number> {
/// Casts from self reference to RootsStorage reference.
fn as_roots_storage(&self) -> &dyn RootsStorage<H, Number>;
/// Execute given functor with cached entry for given trie root.
/// Returns true if the functor has been called (cache entry exists) and false otherwise.
fn with_cached_changed_keys(
&self,
root: &H::Out,
functor: &mut dyn FnMut(&HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>),
) -> bool;
/// Get a trie node.
fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String>;
}
/// Changes trie storage -> trie backend essence adapter.
pub struct TrieBackendStorageAdapter<'a, H: Hasher, Number: BlockNumber>(pub &'a dyn Storage<H, Number>);
impl<'a, H: Hasher, N: BlockNumber> crate::TrieBackendStorage<H> for TrieBackendStorageAdapter<'a, H, N> {
type Overlay = trie::MemoryDB<H>;
fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
self.0.get(key, prefix)
}
}
/// Changes trie configuration.
pub type Configuration = primitives::ChangesTrieConfiguration;
/// Blocks range where configuration has been constant.
#[derive(Clone)]
pub struct ConfigurationRange<'a, N> {
/// Active configuration.
pub config: &'a Configuration,
/// Zero block of this configuration. The configuration is active starting from the next block.
pub zero: N,
/// End block of this configuration. It is the last block where configuration has been active.
pub end: Option<N>,
}
/// Compute the changes trie root and transaction for given block.
/// Returns Err(()) if unknown `parent_hash` has been passed.
/// Returns Ok(None) if there's no data to perform computation.
/// Panics if background storage returns an error OR if insert to MemoryDB fails.
pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, Number: BlockNumber>(
backend: &B,
storage: Option<&'a S>,
changes: &OverlayedChanges,
parent_hash: H::Out,
) -> Result<Option<(MemoryDB<H>, H::Out, CacheAction<H::Out, Number>)>, ()>
where
H::Out: Ord + 'static,
{
let (storage, config) = match (storage, changes.changes_trie_config.as_ref()) {
(Some(storage), Some(config)) => (storage, config),
_ => return Ok(None),
};
// FIXME: remove this in https://github.com/paritytech/substrate/pull/3201
let config = ConfigurationRange {
config,
zero: Zero::zero(),
end: None,
};
// build_anchor error should not be considered fatal
let parent = storage.build_anchor(parent_hash).map_err(|_| ())?;
let block = parent.number.clone() + One::one();
// storage errors are considered fatal (similar to situations when runtime fetches values from storage)
let (input_pairs, child_input_pairs, digest_input_blocks) = prepare_input::<B, H, Number>(
backend,
storage,
config.clone(),
changes,
&parent,
).expect("changes trie: storage access is not allowed to fail within runtime");
// prepare cached data
let mut cache_action = prepare_cached_build_data(config, block.clone());
let needs_changed_keys = cache_action.collects_changed_keys();
cache_action = cache_action.set_digest_input_blocks(digest_input_blocks);
let mut mdb = MemoryDB::default();
let mut child_roots = Vec::with_capacity(child_input_pairs.len());
for (child_index, input_pairs) in child_input_pairs {
let mut not_empty = false;
let mut root = Default::default();
{
let mut trie = TrieDBMut::<H>::new(&mut mdb, &mut root);
let mut storage_changed_keys = HashSet::new();
for input_pair in input_pairs {
if needs_changed_keys {
if let Some(key) = input_pair.key() {
storage_changed_keys.insert(key.to_vec());
}
}
let (key, value) = input_pair.into();
not_empty = true;
trie.insert(&key, &value)
.expect("changes trie: insertion to trie is not allowed to fail within runtime");
}
cache_action = cache_action.insert(
Some(child_index.storage_key.clone()),
storage_changed_keys,
);
}
if not_empty {
child_roots.push(input::InputPair::ChildIndex(child_index, root.as_ref().to_vec()));
}
}
let mut root = Default::default();
{
let mut trie = TrieDBMut::<H>::new(&mut mdb, &mut root);
for (key, value) in child_roots.into_iter().map(Into::into) {
trie.insert(&key, &value)
.expect("changes trie: insertion to trie is not allowed to fail within runtime");
}
let mut storage_changed_keys = HashSet::new();
for input_pair in input_pairs {
if needs_changed_keys {
if let Some(key) = input_pair.key() {
storage_changed_keys.insert(key.to_vec());
}
}
let (key, value) = input_pair.into();
trie.insert(&key, &value)
.expect("changes trie: insertion to trie is not allowed to fail within runtime");
}
cache_action = cache_action.insert(
None,
storage_changed_keys,
);
}
let cache_action = cache_action.complete(block, &root);
Ok(Some((mdb, root, cache_action)))
}
/// Prepare empty cached build data for given block.
fn prepare_cached_build_data<Number: BlockNumber>(
config: ConfigurationRange<Number>,
block: Number,
) -> IncompleteCacheAction<Number> {
// when digests are not enabled in configuration, we do not need to cache anything
// because it'll never be used again for building other tries
// => let's clear the cache
if !config.config.is_digest_build_enabled() {
return IncompleteCacheAction::Clear;
}
// when this is the last block where current configuration is active
// => let's clear the cache
if config.end.as_ref() == Some(&block) {
return IncompleteCacheAction::Clear;
}
// we do not need to cache anything when top-level digest trie is created, because
// it'll never be used again for building other tries
// => let's clear the cache
match config.config.digest_level_at_block(config.zero.clone(), block) {
Some((digest_level, _, _)) if digest_level == config.config.digest_levels => IncompleteCacheAction::Clear,
_ => IncompleteCacheAction::CacheBuildData(IncompleteCachedBuildData::new()),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn cache_is_cleared_when_digests_are_disabled() {
let config = Configuration { digest_interval: 0, digest_levels: 0 };
let config_range = ConfigurationRange { zero: 0, end: None, config: &config };
assert_eq!(prepare_cached_build_data(config_range, 8u32), IncompleteCacheAction::Clear);
}
#[test]
fn build_data_is_cached_when_digests_are_enabled() {
let config = Configuration { digest_interval: 8, digest_levels: 2 };
let config_range = ConfigurationRange { zero: 0, end: None, config: &config };
assert!(prepare_cached_build_data(config_range.clone(), 4u32).collects_changed_keys());
assert!(prepare_cached_build_data(config_range.clone(), 7u32).collects_changed_keys());
assert!(prepare_cached_build_data(config_range, 8u32).collects_changed_keys());
}
#[test]
fn cache_is_cleared_when_digests_are_enabled_and_top_level_digest_is_built() {
let config = Configuration { digest_interval: 8, digest_levels: 2 };
let config_range = ConfigurationRange { zero: 0, end: None, config: &config };
assert_eq!(prepare_cached_build_data(config_range, 64u32), IncompleteCacheAction::Clear);
}
#[test]
fn cache_is_cleared_when_end_block_of_configuration_is_built() {
let config = Configuration { digest_interval: 8, digest_levels: 2 };
let config_range = ConfigurationRange { zero: 0, end: Some(4u32), config: &config };
assert_eq!(prepare_cached_build_data(config_range.clone(), 4u32), IncompleteCacheAction::Clear);
}
}
substrate/primitives/state-machine/src/changes_trie/prune.rs
+370
View File
@@ -0,0 +1,370 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Changes trie pruning-related functions.
use hash_db::Hasher;
use trie::Recorder;
use log::warn;
use num_traits::{One, Zero};
use crate::proving_backend::ProvingBackendRecorder;
use crate::trie_backend_essence::TrieBackendEssence;
use crate::changes_trie::{AnchorBlockId, Configuration, Storage, BlockNumber};
use crate::changes_trie::storage::TrieBackendAdapter;
use crate::changes_trie::input::{ChildIndex, InputKey};
use codec::Decode;
/// Get number of oldest block for which changes trie is not pruned
/// given changes trie configuration, pruning parameter and number of
/// best finalized block.
pub fn oldest_non_pruned_trie<Number: BlockNumber>(
config: &Configuration,
min_blocks_to_keep: Number,
best_finalized_block: Number,
) -> Number {
let max_digest_interval = config.max_digest_interval();
let best_finalized_block_rem = best_finalized_block.clone() % max_digest_interval.into();
let max_digest_block = best_finalized_block - best_finalized_block_rem;
match pruning_range(config, min_blocks_to_keep, max_digest_block) {
Some((_, last_pruned_block)) => last_pruned_block + One::one(),
None => One::one(),
}
}
/// Prune obsolete changes tries. Pruning happens at the same block, where highest
/// level digest is created. Pruning guarantees to save changes tries for last
/// `min_blocks_to_keep` blocks. We only prune changes tries at `max_digest_interval`
/// ranges.
/// Returns MemoryDB that contains all deleted changes tries nodes.
pub fn prune<S: Storage<H, Number>, H: Hasher, Number: BlockNumber, F: FnMut(H::Out)>(
config: &Configuration,
storage: &S,
min_blocks_to_keep: Number,
current_block: &AnchorBlockId<H::Out, Number>,
mut remove_trie_node: F,
) {
// select range for pruning
let (first, last) = match pruning_range(config, min_blocks_to_keep, current_block.number.clone()) {
Some((first, last)) => (first, last),
None => return,
};
// delete changes trie for every block in range
// FIXME: limit `max_digest_interval` so that this cycle won't involve huge ranges
let mut block = first;
loop {
if block >= last.clone() + One::one() {
break;
}
let prev_block = block.clone();
block += One::one();
let block = prev_block;
let root = match storage.root(current_block, block.clone()) {
Ok(Some(root)) => root,
Ok(None) => continue,
Err(error) => {
// try to delete other tries
warn!(target: "trie", "Failed to read changes trie root from DB: {}", error);
continue;
},
};
let children_roots = {
let trie_storage = TrieBackendEssence::<_, H>::new(
crate::changes_trie::TrieBackendStorageAdapter(storage),
root,
);
let child_prefix = ChildIndex::key_neutral_prefix(block.clone());
let mut children_roots = Vec::new();
trie_storage.for_key_values_with_prefix(&child_prefix, |key, value| {
if let Ok(InputKey::ChildIndex::<Number>(_trie_key)) = Decode::decode(&mut &key[..]) {
if let Ok(value) = <Vec<u8>>::decode(&mut &value[..]) {
let mut trie_root = <H as Hasher>::Out::default();
trie_root.as_mut().copy_from_slice(&value[..]);
children_roots.push(trie_root);
}
}
});
children_roots
};
for root in children_roots.into_iter() {
prune_trie(storage, root, &mut remove_trie_node);
}
prune_trie(storage, root, &mut remove_trie_node);
}
}
// Prune a trie.
fn prune_trie<S: Storage<H, Number>, H: Hasher, Number: BlockNumber, F: FnMut(H::Out)>(
storage: &S,
root: H::Out,
remove_trie_node: &mut F,
) {
// enumerate all changes trie' keys, recording all nodes that have been 'touched'
// (effectively - all changes trie nodes)
let mut proof_recorder: Recorder<H::Out> = Default::default();
{
let mut trie = ProvingBackendRecorder::<_, H> {
backend: &TrieBackendEssence::new(TrieBackendAdapter::new(storage), root),
proof_recorder: &mut proof_recorder,
};
trie.record_all_keys();
}
// all nodes of this changes trie should be pruned
remove_trie_node(root);
for node in proof_recorder.drain().into_iter().map(|n| n.hash) {
remove_trie_node(node);
}
}
/// Select blocks range (inclusive from both ends) for pruning changes tries in.
fn pruning_range<Number: BlockNumber>(
config: &Configuration,
min_blocks_to_keep: Number,
block: Number,
) -> Option<(Number, Number)> {
// compute number of changes tries we actually want to keep
let (prune_interval, blocks_to_keep) = if config.is_digest_build_enabled() {
// we only CAN prune at block where max-level-digest is created
let max_digest_interval = match config.digest_level_at_block(Zero::zero(), block.clone()) {
Some((digest_level, digest_interval, _)) if digest_level == config.digest_levels =>
digest_interval,
_ => return None,
};
// compute maximal number of high-level digests to keep
let max_digest_intervals_to_keep = max_digest_intervals_to_keep(min_blocks_to_keep, max_digest_interval);
// number of blocks BEFORE current block where changes tries are not pruned
(
max_digest_interval,
max_digest_intervals_to_keep.checked_mul(&max_digest_interval.into())
)
} else {
(
1,
Some(min_blocks_to_keep)
)
};
// last block for which changes trie is pruned
let last_block_to_prune = blocks_to_keep.and_then(|b| block.checked_sub(&b));
let first_block_to_prune = last_block_to_prune
.clone()
.and_then(|b| b.checked_sub(&prune_interval.into()));
last_block_to_prune
.and_then(|last| first_block_to_prune.map(|first| (first + One::one(), last)))
}
/// Select pruning delay for the changes tries. To make sure we could build a changes
/// trie at block B, we need an access to previous:
/// max_digest_interval = config.digest_interval ^ config.digest_levels
/// blocks. So we can only prune blocks that are earlier than B - max_digest_interval.
/// The pruning_delay stands for number of max_digest_interval-s that we want to keep:
/// 0 or 1: means that only last changes trie is guaranteed to exists;
/// 2: the last changes trie + previous changes trie
/// ...
fn max_digest_intervals_to_keep<Number: BlockNumber>(
min_blocks_to_keep: Number,
max_digest_interval: u32,
) -> Number {
// config.digest_level_at_block ensures that it is not zero
debug_assert!(max_digest_interval != 0);
let max_digest_intervals_to_keep = min_blocks_to_keep / max_digest_interval.into();
if max_digest_intervals_to_keep.is_zero() {
One::one()
} else {
max_digest_intervals_to_keep
}
}
#[cfg(test)]
mod tests {
use std::collections::HashSet;
use trie::MemoryDB;
use primitives::Blake2Hasher;
use crate::backend::insert_into_memory_db;
use crate::changes_trie::storage::InMemoryStorage;
use codec::Encode;
use super::*;
fn config(interval: u32, levels: u32) -> Configuration {
Configuration {
digest_interval: interval,
digest_levels: levels,
}
}
fn prune_by_collect<S: Storage<H, u64>, H: Hasher>(
config: &Configuration,
storage: &S,
min_blocks_to_keep: u64,
current_block: u64,
) -> HashSet<H::Out> {
let mut pruned_trie_nodes = HashSet::new();
prune(config,
storage,
min_blocks_to_keep,
&AnchorBlockId { hash: Default::default(), number: current_block },
|node| { pruned_trie_nodes.insert(node); });
pruned_trie_nodes
}
#[test]
fn prune_works() {
fn prepare_storage() -> InMemoryStorage<Blake2Hasher, u64> {
let child_key = ChildIndex { block: 67u64, storage_key: b"1".to_vec() }.encode();
let mut mdb1 = MemoryDB::<Blake2Hasher>::default();
let root1 = insert_into_memory_db::<Blake2Hasher, _>(
&mut mdb1, vec![(vec![10], vec![20])]).unwrap();
let mut mdb2 = MemoryDB::<Blake2Hasher>::default();
let root2 = insert_into_memory_db::<Blake2Hasher, _>(
&mut mdb2, vec![(vec![11], vec![21]), (vec![12], vec![22])]).unwrap();
let mut mdb3 = MemoryDB::<Blake2Hasher>::default();
let ch_root3 = insert_into_memory_db::<Blake2Hasher, _>(
&mut mdb3, vec![(vec![110], vec![120])]).unwrap();
let root3 = insert_into_memory_db::<Blake2Hasher, _>(&mut mdb3, vec![
(vec![13], vec![23]),
(vec![14], vec![24]),
(child_key, ch_root3.as_ref().encode()),
]).unwrap();
let mut mdb4 = MemoryDB::<Blake2Hasher>::default();
let root4 = insert_into_memory_db::<Blake2Hasher, _>(
&mut mdb4, vec![(vec![15], vec![25])]).unwrap();
let storage = InMemoryStorage::new();
storage.insert(65, root1, mdb1);
storage.insert(66, root2, mdb2);
storage.insert(67, root3, mdb3);
storage.insert(68, root4, mdb4);
storage
}
// l1-digest is created every 2 blocks
// l2-digest is created every 4 blocks
// we do not want to keep any additional changes tries
// => only one l2-digest is saved AND it is pruned once next is created
let config = Configuration { digest_interval: 2, digest_levels: 2 };
let storage = prepare_storage();
assert!(prune_by_collect(&config, &storage, 0, 69).is_empty());
assert!(prune_by_collect(&config, &storage, 0, 70).is_empty());
assert!(prune_by_collect(&config, &storage, 0, 71).is_empty());
let non_empty = prune_by_collect(&config, &storage, 0, 72);
assert!(!non_empty.is_empty());
storage.remove_from_storage(&non_empty);
assert!(storage.into_mdb().drain().is_empty());
// l1-digest is created every 2 blocks
// l2-digest is created every 4 blocks
// we want keep 1 additional changes tries
let config = Configuration { digest_interval: 2, digest_levels: 2 };
let storage = prepare_storage();
assert!(prune_by_collect(&config, &storage, 8, 69).is_empty());
assert!(prune_by_collect(&config, &storage, 8, 70).is_empty());
assert!(prune_by_collect(&config, &storage, 8, 71).is_empty());
assert!(prune_by_collect(&config, &storage, 8, 72).is_empty());
assert!(prune_by_collect(&config, &storage, 8, 73).is_empty());
assert!(prune_by_collect(&config, &storage, 8, 74).is_empty());
assert!(prune_by_collect(&config, &storage, 8, 75).is_empty());
let non_empty = prune_by_collect(&config, &storage, 8, 76);
assert!(!non_empty.is_empty());
storage.remove_from_storage(&non_empty);
assert!(storage.into_mdb().drain().is_empty());
// l1-digest is created every 2 blocks
// we want keep 2 additional changes tries
let config = Configuration { digest_interval: 2, digest_levels: 1 };
let storage = prepare_storage();
assert!(prune_by_collect(&config, &storage, 4, 69).is_empty());
let non_empty = prune_by_collect(&config, &storage, 4, 70);
assert!(!non_empty.is_empty());
storage.remove_from_storage(&non_empty);
assert!(prune_by_collect(&config, &storage, 4, 71).is_empty());
let non_empty = prune_by_collect(&config, &storage, 4, 72);
assert!(!non_empty.is_empty());
storage.remove_from_storage(&non_empty);
assert!(storage.into_mdb().drain().is_empty());
}
#[test]
fn pruning_range_works() {
// DIGESTS ARE NOT CREATED + NO TRIES ARE PRUNED
assert_eq!(pruning_range(&config(10, 0), 2u64, 2u64), None);
// DIGESTS ARE NOT CREATED + SOME TRIES ARE PRUNED
assert_eq!(pruning_range(&config(10, 0), 100u64, 110u64), Some((10, 10)));
assert_eq!(pruning_range(&config(10, 0), 100u64, 210u64), Some((110, 110)));
// DIGESTS ARE CREATED + NO TRIES ARE PRUNED
assert_eq!(pruning_range(&config(10, 2), 2u64, 0u64), None);
assert_eq!(pruning_range(&config(10, 2), 30u64, 100u64), None);
assert_eq!(pruning_range(&config(::std::u32::MAX, 2), 1u64, 1024u64), None);
assert_eq!(pruning_range(&config(::std::u32::MAX, 2), ::std::u64::MAX, 1024u64), None);
assert_eq!(pruning_range(&config(32, 2), 2048u64, 512u64), None);
assert_eq!(pruning_range(&config(32, 2), 2048u64, 1024u64), None);
// DIGESTS ARE CREATED + SOME TRIES ARE PRUNED
// when we do not want to keep any highest-level-digests
// (system forces to keep at least one)
assert_eq!(pruning_range(&config(4, 2), 0u64, 32u64), Some((1, 16)));
assert_eq!(pruning_range(&config(4, 2), 0u64, 64u64), Some((33, 48)));
// when we want to keep 1 (last) highest-level-digest
assert_eq!(pruning_range(&config(4, 2), 16u64, 32u64), Some((1, 16)));
assert_eq!(pruning_range(&config(4, 2), 16u64, 64u64), Some((33, 48)));
// when we want to keep 1 (last) + 1 additional level digests
assert_eq!(pruning_range(&config(32, 2), 4096u64, 5120u64), Some((1, 1024)));
assert_eq!(pruning_range(&config(32, 2), 4096u64, 6144u64), Some((1025, 2048)));
}
#[test]
fn max_digest_intervals_to_keep_works() {
assert_eq!(max_digest_intervals_to_keep(1024u64, 1025), 1u64);
assert_eq!(max_digest_intervals_to_keep(1024u64, 1023), 1u64);
assert_eq!(max_digest_intervals_to_keep(1024u64, 512), 2u64);
assert_eq!(max_digest_intervals_to_keep(1024u64, 511), 2u64);
assert_eq!(max_digest_intervals_to_keep(1024u64, 100), 10u64);
}
#[test]
fn oldest_non_pruned_trie_works() {
// when digests are not created at all
assert_eq!(oldest_non_pruned_trie(&config(0, 0), 100u64, 10u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(0, 0), 100u64, 110u64), 11);
// when only l1 digests are created
assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100u64, 50u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100u64, 110u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100u64, 210u64), 101);
// when l2 digests are created
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 50u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 110u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 210u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 10110u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 20110u64), 10001);
}
}
substrate/primitives/state-machine/src/changes_trie/storage.rs
+208
View File
@@ -0,0 +1,208 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Changes trie storage utilities.
use std::collections::{BTreeMap, HashSet, HashMap};
use hash_db::{Hasher, Prefix, EMPTY_PREFIX};
use trie::DBValue;
use trie::MemoryDB;
use parking_lot::RwLock;
use crate::changes_trie::{BuildCache, RootsStorage, Storage, AnchorBlockId, BlockNumber};
use crate::trie_backend_essence::TrieBackendStorage;
#[cfg(test)]
use crate::backend::insert_into_memory_db;
#[cfg(test)]
use crate::changes_trie::input::{InputPair, ChildIndex};
/// In-memory implementation of changes trie storage.
pub struct InMemoryStorage<H: Hasher, Number: BlockNumber> {
data: RwLock<InMemoryStorageData<H, Number>>,
cache: BuildCache<H::Out, Number>,
}
/// Adapter for using changes trie storage as a TrieBackendEssence' storage.
pub struct TrieBackendAdapter<'a, H: Hasher, Number: BlockNumber> {
storage: &'a dyn Storage<H, Number>,
_hasher: ::std::marker::PhantomData<(H, Number)>,
}
struct InMemoryStorageData<H: Hasher, Number: BlockNumber> {
roots: BTreeMap<Number, H::Out>,
mdb: MemoryDB<H>,
}
impl<H: Hasher, Number: BlockNumber> InMemoryStorage<H, Number> {
/// Creates storage from given in-memory database.
pub fn with_db(mdb: MemoryDB<H>) -> Self {
Self {
data: RwLock::new(InMemoryStorageData {
roots: BTreeMap::new(),
mdb,
}),
cache: BuildCache::new(),
}
}
/// Creates storage with empty database.
pub fn new() -> Self {
Self::with_db(Default::default())
}
/// Creates storage with given proof.
pub fn with_proof(proof: Vec<Vec<u8>>) -> Self {
use hash_db::HashDB;
let mut proof_db = MemoryDB::<H>::default();
for item in proof {
proof_db.insert(EMPTY_PREFIX, &item);
}
Self::with_db(proof_db)
}
/// Get mutable cache reference.
pub fn cache_mut(&mut self) -> &mut BuildCache<H::Out, Number> {
&mut self.cache
}
/// Create the storage with given blocks.
pub fn with_blocks(blocks: Vec<(Number, H::Out)>) -> Self {
Self {
data: RwLock::new(InMemoryStorageData {
roots: blocks.into_iter().collect(),
mdb: MemoryDB::default(),
}),
cache: BuildCache::new(),
}
}
#[cfg(test)]
pub fn with_inputs(
mut top_inputs: Vec<(Number, Vec<InputPair<Number>>)>,
children_inputs: Vec<(Vec<u8>, Vec<(Number, Vec<InputPair<Number>>)>)>,
) -> Self {
let mut mdb = MemoryDB::default();
let mut roots = BTreeMap::new();
for (storage_key, child_input) in children_inputs {
for (block, pairs) in child_input {
let root = insert_into_memory_db::<H, _>(&mut mdb, pairs.into_iter().map(Into::into));
if let Some(root) = root {
let ix = if let Some(ix) = top_inputs.iter().position(|v| v.0 == block) {
ix
} else {
top_inputs.push((block.clone(), Default::default()));
top_inputs.len() - 1
};
top_inputs[ix].1.push(InputPair::ChildIndex(
ChildIndex { block: block.clone(), storage_key: storage_key.clone() },
root.as_ref().to_vec(),
));
}
}
}
for (block, pairs) in top_inputs {
let root = insert_into_memory_db::<H, _>(&mut mdb, pairs.into_iter().map(Into::into));
if let Some(root) = root {
roots.insert(block, root);
}
}
InMemoryStorage {
data: RwLock::new(InMemoryStorageData {
roots,
mdb,
}),
cache: BuildCache::new(),
}
}
#[cfg(test)]
pub fn clear_storage(&self) {
self.data.write().mdb = MemoryDB::default(); // use new to be more correct
}
#[cfg(test)]
pub fn remove_from_storage(&self, keys: &HashSet<H::Out>) {
let mut data = self.data.write();
for key in keys {
data.mdb.remove_and_purge(key, hash_db::EMPTY_PREFIX);
}
}
#[cfg(test)]
pub fn into_mdb(self) -> MemoryDB<H> {
self.data.into_inner().mdb
}
/// Insert changes trie for given block.
pub fn insert(&self, block: Number, changes_trie_root: H::Out, trie: MemoryDB<H>) {
let mut data = self.data.write();
data.roots.insert(block, changes_trie_root);
data.mdb.consolidate(trie);
}
}
impl<H: Hasher, Number: BlockNumber> RootsStorage<H, Number> for InMemoryStorage<H, Number> {
fn build_anchor(&self, parent_hash: H::Out) -> Result<AnchorBlockId<H::Out, Number>, String> {
self.data.read().roots.iter()
.find(|(_, v)| **v == parent_hash)
.map(|(k, _)| AnchorBlockId { hash: parent_hash, number: k.clone() })
.ok_or_else(|| format!("Can't find associated number for block {:?}", parent_hash))
}
fn root(&self, _anchor_block: &AnchorBlockId<H::Out, Number>, block: Number) -> Result<Option<H::Out>, String> {
Ok(self.data.read().roots.get(&block).cloned())
}
}
impl<H: Hasher, Number: BlockNumber> Storage<H, Number> for InMemoryStorage<H, Number> {
fn as_roots_storage(&self) -> &dyn RootsStorage<H, Number> {
self
}
fn with_cached_changed_keys(
&self,
root: &H::Out,
functor: &mut dyn FnMut(&HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>),
) -> bool {
self.cache.with_changed_keys(root, functor)
}
fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
MemoryDB::<H>::get(&self.data.read().mdb, key, prefix)
}
}
impl<'a, H: Hasher, Number: BlockNumber> TrieBackendAdapter<'a, H, Number> {
pub fn new(storage: &'a dyn Storage<H, Number>) -> Self {
Self { storage, _hasher: Default::default() }
}
}
impl<'a, H, Number> TrieBackendStorage<H> for TrieBackendAdapter<'a, H, Number>
where
Number: BlockNumber,
H: Hasher,
{
type Overlay = MemoryDB<H>;
fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
self.storage.get(key, prefix)
}
}
substrate/primitives/state-machine/src/changes_trie/surface_iterator.rs
+285
View File
@@ -0,0 +1,285 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! The best way to understand how this iterator works is to imagine some 2D terrain that have some mountains
//! (digest changes tries) and valleys (changes tries for regular blocks). There are gems (blocks) beneath the
//! terrain. Given the request to find all gems in the range [X1; X2] this iterator will return **minimal set**
//! of points at the terrain (mountains and valleys) inside this range that have to be drilled down to
//! search for gems.
use num_traits::One;
use crate::changes_trie::{ConfigurationRange, BlockNumber};
/// Returns surface iterator for given range of blocks.
///
/// `max` is the number of best block, known to caller. We can't access any changes tries
/// that are built after this block, even though we may have them built already.
pub fn surface_iterator<'a, Number: BlockNumber>(
config: ConfigurationRange<'a, Number>,
max: Number,
begin: Number,
end: Number,
) -> Result<SurfaceIterator<'a, Number>, String> {
let (current, current_begin, digest_step, digest_level) = lower_bound_max_digest(
config.clone(),
max.clone(),
begin.clone(),
end,
)?;
Ok(SurfaceIterator {
config,
begin,
max,
current: Some(current),
current_begin,
digest_step,
digest_level,
})
}
/// Surface iterator - only traverses top-level digests from given range and tries to find
/// all valid digest changes.
///
/// Iterator item is the tuple of (last block of the current point + digest level of the current point).
/// Digest level is Some(0) when it is regular block, is Some(non-zero) when it is digest block and None
/// if it is skewed digest block.
pub struct SurfaceIterator<'a, Number: BlockNumber> {
config: ConfigurationRange<'a, Number>,
begin: Number,
max: Number,
current: Option<Number>,
current_begin: Number,
digest_step: u32,
digest_level: Option<u32>,
}
impl<'a, Number: BlockNumber> Iterator for SurfaceIterator<'a, Number> {
type Item = Result<(Number, Option<u32>), String>;
fn next(&mut self) -> Option<Self::Item> {
let current = self.current.clone()?;
let digest_level = self.digest_level;
if current < self.digest_step.into() {
self.current = None;
} else {
let next = current.clone() - self.digest_step.into();
if next.is_zero() || next < self.begin {
self.current = None;
} else if next > self.current_begin {
self.current = Some(next);
} else {
let max_digest_interval = lower_bound_max_digest(
self.config.clone(),
self.max.clone(),
self.begin.clone(),
next,
);
let (current, current_begin, digest_step, digest_level) = match max_digest_interval {
Err(err) => return Some(Err(err)),
Ok(range) => range,
};
self.current = Some(current);
self.current_begin = current_begin;
self.digest_step = digest_step;
self.digest_level = digest_level;
}
}
Some(Ok((current, digest_level)))
}
}
/// Returns parameters of highest level digest block that includes the end of given range
/// and tends to include the whole range.
fn lower_bound_max_digest<'a, Number: BlockNumber>(
config: ConfigurationRange<'a, Number>,
max: Number,
begin: Number,
end: Number,
) -> Result<(Number, Number, u32, Option<u32>), String> {
if end > max || begin > end {
return Err(format!("invalid changes range: {}..{}/{}", begin, end, max));
}
if begin <= config.zero || config.end.as_ref().map(|config_end| end > *config_end).unwrap_or(false) {
return Err(format!("changes trie range is not covered by configuration: {}..{}/{}..{}",
begin, end, config.zero, match config.end.as_ref() {
Some(config_end) => format!("{}", config_end),
None => "None".into(),
}));
}
let mut digest_level = 0u32;
let mut digest_step = 1u32;
let mut digest_interval = 0u32;
let mut current = end.clone();
let mut current_begin = begin.clone();
if current_begin != current {
while digest_level != config.config.digest_levels {
// try to use next level digest
let new_digest_level = digest_level + 1;
let new_digest_step = digest_step * config.config.digest_interval;
let new_digest_interval = config.config.digest_interval * {
if digest_interval == 0 { 1 } else { digest_interval }
};
let new_digest_begin = config.zero.clone() + ((current.clone() - One::one() - config.zero.clone())
/ new_digest_interval.into()) * new_digest_interval.into();
let new_digest_end = new_digest_begin.clone() + new_digest_interval.into();
let new_current = new_digest_begin.clone() + new_digest_interval.into();
// check if we met skewed digest
if let Some(skewed_digest_end) = config.end.as_ref() {
if new_digest_end > *skewed_digest_end {
let skewed_digest_start = config.config.prev_max_level_digest_block(
config.zero.clone(),
skewed_digest_end.clone(),
);
if let Some(skewed_digest_start) = skewed_digest_start {
let skewed_digest_range = (skewed_digest_end.clone() - skewed_digest_start.clone())
.try_into().ok()
.expect("skewed digest range is always <= max level digest range;\
max level digest range always fits u32; qed");
return Ok((
skewed_digest_end.clone(),
skewed_digest_start,
skewed_digest_range,
None,
));
}
}
}
// we can't use next level digest if it touches any unknown (> max) blocks
if new_digest_end > max {
if begin < new_digest_begin {
current_begin = new_digest_begin;
}
break;
}
// we can (and will) use this digest
digest_level = new_digest_level;
digest_step = new_digest_step;
digest_interval = new_digest_interval;
current = new_current;
current_begin = new_digest_begin;
// if current digest covers the whole range => no need to use next level digest
if current_begin <= begin && new_digest_end >= end {
break;
}
}
}
Ok((
current,
current_begin,
digest_step,
Some(digest_level),
))
}
#[cfg(test)]
mod tests {
use crate::changes_trie::{Configuration};
use super::*;
fn configuration_range<'a>(config: &'a Configuration, zero: u64) -> ConfigurationRange<'a, u64> {
ConfigurationRange {
config,
zero,
end: None,
}
}
#[test]
fn lower_bound_max_digest_works() {
let config = Configuration { digest_interval: 4, digest_levels: 2 };
// when config activates at 0
assert_eq!(
lower_bound_max_digest(configuration_range(&config, 0u64), 100_000u64, 20u64, 180u64).unwrap(),
(192, 176, 16, Some(2)),
);
// when config activates at 30
assert_eq!(
lower_bound_max_digest(configuration_range(&config, 30u64), 100_000u64, 50u64, 210u64).unwrap(),
(222, 206, 16, Some(2)),
);
}
#[test]
fn surface_iterator_works() {
let config = Configuration { digest_interval: 4, digest_levels: 2 };
// when config activates at 0
assert_eq!(
surface_iterator(
configuration_range(&config, 0u64),
100_000u64,
40u64,
180u64,
).unwrap().collect::<Vec<_>>(),
vec![
Ok((192, Some(2))), Ok((176, Some(2))), Ok((160, Some(2))), Ok((144, Some(2))),
Ok((128, Some(2))), Ok((112, Some(2))), Ok((96, Some(2))), Ok((80, Some(2))),
Ok((64, Some(2))), Ok((48, Some(2))),
],
);
// when config activates at 30
assert_eq!(
surface_iterator(
configuration_range(&config, 30u64),
100_000u64,
40u64,
180u64,
).unwrap().collect::<Vec<_>>(),
vec![
Ok((190, Some(2))), Ok((174, Some(2))), Ok((158, Some(2))), Ok((142, Some(2))), Ok((126, Some(2))),
Ok((110, Some(2))), Ok((94, Some(2))), Ok((78, Some(2))), Ok((62, Some(2))), Ok((46, Some(2))),
],
);
// when config activates at 0 AND max block is before next digest
assert_eq!(
surface_iterator(configuration_range(&config, 0u64), 183u64, 40u64, 183u64).unwrap().collect::<Vec<_>>(),
vec![
Ok((183, Some(0))), Ok((182, Some(0))), Ok((181, Some(0))), Ok((180, Some(1))),
Ok((176, Some(2))), Ok((160, Some(2))), Ok((144, Some(2))), Ok((128, Some(2))), Ok((112, Some(2))),
Ok((96, Some(2))), Ok((80, Some(2))), Ok((64, Some(2))), Ok((48, Some(2))),
],
);
}
#[test]
fn surface_iterator_works_with_skewed_digest() {
let config = Configuration { digest_interval: 4, digest_levels: 2 };
let mut config_range = configuration_range(&config, 0u64);
// when config activates at 0 AND ends at 170
config_range.end = Some(170);
assert_eq!(
surface_iterator(config_range, 100_000u64, 40u64, 170u64).unwrap().collect::<Vec<_>>(),
vec![
Ok((170, None)), Ok((160, Some(2))), Ok((144, Some(2))), Ok((128, Some(2))), Ok((112, Some(2))),
Ok((96, Some(2))), Ok((80, Some(2))), Ok((64, Some(2))), Ok((48, Some(2))),
],
);
}
}
substrate/primitives/state-machine/src/error.rs
+47
View File
@@ -0,0 +1,47 @@
// Copyright 2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
/// State Machine Errors
use std::fmt;
/// State Machine Error bound.
///
/// This should reflect Wasm error type bound for future compatibility.
pub trait Error: 'static + fmt::Debug + fmt::Display + Send {}
impl<T: 'static + fmt::Debug + fmt::Display + Send> Error for T {}
/// Externalities Error.
///
/// Externalities are not really allowed to have errors, since it's assumed that dependent code
/// would not be executed unless externalities were available. This is included for completeness,
/// and as a transition away from the pre-existing framework.
#[derive(Debug, Eq, PartialEq)]
pub enum ExecutionError {
/// Backend error.
Backend(String),
/// The entry `:code` doesn't exist in storage so there's no way we can execute anything.
CodeEntryDoesNotExist,
/// Backend is incompatible with execution proof generation process.
UnableToGenerateProof,
/// Invalid execution proof.
InvalidProof,
}
impl fmt::Display for ExecutionError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "Externalities Error") }
}
substrate/primitives/state-machine/src/ext.rs
+606
View File
@@ -0,0 +1,606 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Concrete externalities implementation.
use crate::{
backend::Backend, OverlayedChanges,
changes_trie::{
Storage as ChangesTrieStorage, CacheAction as ChangesTrieCacheAction, build_changes_trie,
},
};
use hash_db::Hasher;
use primitives::{
storage::{ChildStorageKey, well_known_keys::is_child_storage_key},
traits::Externalities, hexdisplay::HexDisplay, hash::H256,
};
use trie::{trie_types::Layout, MemoryDB, default_child_trie_root};
use externalities::Extensions;
use std::{error, fmt, any::{Any, TypeId}};
use log::{warn, trace};
const EXT_NOT_ALLOWED_TO_FAIL: &str = "Externalities not allowed to fail within runtime";
/// Errors that can occur when interacting with the externalities.
#[derive(Debug, Copy, Clone)]
pub enum Error<B, E> {
/// Failure to load state data from the backend.
#[allow(unused)]
Backend(B),
/// Failure to execute a function.
#[allow(unused)]
Executor(E),
}
impl<B: fmt::Display, E: fmt::Display> fmt::Display for Error<B, E> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
Error::Backend(ref e) => write!(f, "Storage backend error: {}", e),
Error::Executor(ref e) => write!(f, "Sub-call execution error: {}", e),
}
}
}
impl<B: error::Error, E: error::Error> error::Error for Error<B, E> {
fn description(&self) -> &str {
match *self {
Error::Backend(..) => "backend error",
Error::Executor(..) => "executor error",
}
}
}
/// Wraps a read-only backend, call executor, and current overlayed changes.
pub struct Ext<'a, H, N, B, T> where H: Hasher<Out=H256>, B: 'a + Backend<H> {
/// The overlayed changes to write to.
overlay: &'a mut OverlayedChanges,
/// The storage backend to read from.
backend: &'a B,
/// The storage transaction necessary to commit to the backend. Is cached when
/// `storage_root` is called and the cache is cleared on every subsequent change.
storage_transaction: Option<(B::Transaction, H::Out)>,
/// Changes trie storage to read from.
changes_trie_storage: Option<&'a T>,
/// The changes trie transaction necessary to commit to the changes trie backend.
/// Set to Some when `storage_changes_root` is called. Could be replaced later
/// by calling `storage_changes_root` again => never used as cache.
/// This differs from `storage_transaction` behavior, because the moment when
/// `storage_changes_root` is called matters + we need to remember additional
/// data at this moment (block number).
changes_trie_transaction: Option<(MemoryDB<H>, H::Out, ChangesTrieCacheAction<H::Out, N>)>,
/// Pseudo-unique id used for tracing.
pub id: u16,
/// Dummy usage of N arg.
_phantom: std::marker::PhantomData<N>,
/// Extensions registered with this instance.
extensions: Option<&'a mut Extensions>,
}
impl<'a, H, N, B, T> Ext<'a, H, N, B, T>
where
H: Hasher<Out=H256>,
B: 'a + Backend<H>,
T: 'a + ChangesTrieStorage<H, N>,
N: crate::changes_trie::BlockNumber,
{
/// Create a new `Ext` from overlayed changes and read-only backend
pub fn new(
overlay: &'a mut OverlayedChanges,
backend: &'a B,
changes_trie_storage: Option<&'a T>,
extensions: Option<&'a mut Extensions>,
) -> Self {
Ext {
overlay,
backend,
storage_transaction: None,
changes_trie_storage,
changes_trie_transaction: None,
id: rand::random(),
_phantom: Default::default(),
extensions,
}
}
/// Get the transaction necessary to update the backend.
pub fn transaction(&mut self) -> (
(B::Transaction, H256),
Option<crate::ChangesTrieTransaction<H, N>>,
) {
let _ = self.storage_root();
let (storage_transaction, changes_trie_transaction) = (
self.storage_transaction
.take()
.expect("storage_transaction always set after calling storage root; qed"),
self.changes_trie_transaction
.take()
.map(|(tx, _, cache)| (tx, cache)),
);
(
storage_transaction,
changes_trie_transaction,
)
}
/// Invalidates the currently cached storage root and the db transaction.
///
/// Called when there are changes that likely will invalidate the storage root.
fn mark_dirty(&mut self) {
self.storage_transaction = None;
}
}
#[cfg(test)]
impl<'a, H, N, B, T> Ext<'a, H, N, B, T>
where
H: Hasher<Out=H256>,
B: 'a + Backend<H>,
T: 'a + ChangesTrieStorage<H, N>,
N: crate::changes_trie::BlockNumber,
{
pub fn storage_pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
use std::collections::HashMap;
self.backend.pairs().iter()
.map(|&(ref k, ref v)| (k.to_vec(), Some(v.to_vec())))
.chain(self.overlay.committed.top.clone().into_iter().map(|(k, v)| (k, v.value)))
.chain(self.overlay.prospective.top.clone().into_iter().map(|(k, v)| (k, v.value)))
.collect::<HashMap<_, _>>()
.into_iter()
.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
.collect()
}
}
impl<'a, H, B, T, N> Externalities for Ext<'a, H, N, B, T>
where
H: Hasher<Out=H256>,
B: 'a + Backend<H>,
T: 'a + ChangesTrieStorage<H, N>,
N: crate::changes_trie::BlockNumber,
{
fn storage(&self, key: &[u8]) -> Option<Vec<u8>> {
let _guard = panic_handler::AbortGuard::force_abort();
let result = self.overlay.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(||
self.backend.storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL));
trace!(target: "state-trace", "{:04x}: Get {}={:?}",
self.id,
HexDisplay::from(&key),
result.as_ref().map(HexDisplay::from)
);
result
}
fn storage_hash(&self, key: &[u8]) -> Option<H256> {
let _guard = panic_handler::AbortGuard::force_abort();
let result = self.overlay
.storage(key)
.map(|x| x.map(|x| H::hash(x)))
.unwrap_or_else(||
self.backend.storage_hash(key).expect(EXT_NOT_ALLOWED_TO_FAIL)
);
trace!(target: "state-trace", "{:04x}: Hash {}={:?}",
self.id,
HexDisplay::from(&key),
result,
);
result
}
fn original_storage(&self, key: &[u8]) -> Option<Vec<u8>> {
let _guard = panic_handler::AbortGuard::force_abort();
let result = self.backend.storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL);
trace!(target: "state-trace", "{:04x}: GetOriginal {}={:?}",
self.id,
HexDisplay::from(&key),
result.as_ref().map(HexDisplay::from)
);
result
}
fn original_storage_hash(&self, key: &[u8]) -> Option<H256> {
let _guard = panic_handler::AbortGuard::force_abort();
let result = self.backend.storage_hash(key).expect(EXT_NOT_ALLOWED_TO_FAIL);
trace!(target: "state-trace", "{:04x}: GetOriginalHash {}={:?}",
self.id,
HexDisplay::from(&key),
result,
);
result
}
fn child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option<Vec<u8>> {
let _guard = panic_handler::AbortGuard::force_abort();
let result = self.overlay
.child_storage(storage_key.as_ref(), key)
.map(|x| x.map(|x| x.to_vec()))
.unwrap_or_else(||
self.backend.child_storage(storage_key.as_ref(), key).expect(EXT_NOT_ALLOWED_TO_FAIL)
);
trace!(target: "state-trace", "{:04x}: GetChild({}) {}={:?}",
self.id,
HexDisplay::from(&storage_key.as_ref()),
HexDisplay::from(&key),
result.as_ref().map(HexDisplay::from)
);
result
}
fn child_storage_hash(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option<H256> {
let _guard = panic_handler::AbortGuard::force_abort();
let result = self.overlay
.child_storage(storage_key.as_ref(), key)
.map(|x| x.map(|x| H::hash(x)))
.unwrap_or_else(||
self.backend.storage_hash(key).expect(EXT_NOT_ALLOWED_TO_FAIL)
);
trace!(target: "state-trace", "{:04x}: ChildHash({}) {}={:?}",
self.id,
HexDisplay::from(&storage_key.as_ref()),
HexDisplay::from(&key),
result,
);
result
}
fn original_child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option<Vec<u8>> {
let _guard = panic_handler::AbortGuard::force_abort();
let result = self.backend
.child_storage(storage_key.as_ref(), key)
.expect(EXT_NOT_ALLOWED_TO_FAIL);
trace!(target: "state-trace", "{:04x}: ChildOriginal({}) {}={:?}",
self.id,
HexDisplay::from(&storage_key.as_ref()),
HexDisplay::from(&key),
result.as_ref().map(HexDisplay::from),
);
result
}
fn original_child_storage_hash(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option<H256> {
let _guard = panic_handler::AbortGuard::force_abort();
let result = self.backend
.child_storage_hash(storage_key.as_ref(), key)
.expect(EXT_NOT_ALLOWED_TO_FAIL);
trace!(target: "state-trace", "{}: ChildHashOriginal({}) {}={:?}",
self.id,
HexDisplay::from(&storage_key.as_ref()),
HexDisplay::from(&key),
result,
);
result
}
fn exists_storage(&self, key: &[u8]) -> bool {
let _guard = panic_handler::AbortGuard::force_abort();
let result = match self.overlay.storage(key) {
Some(x) => x.is_some(),
_ => self.backend.exists_storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL),
};
trace!(target: "state-trace", "{:04x}: Exists {}={:?}",
self.id,
HexDisplay::from(&key),
result,
);
result
}
fn exists_child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> bool {
let _guard = panic_handler::AbortGuard::force_abort();
let result = match self.overlay.child_storage(storage_key.as_ref(), key) {
Some(x) => x.is_some(),
_ => self.backend
.exists_child_storage(storage_key.as_ref(), key)
.expect(EXT_NOT_ALLOWED_TO_FAIL),
};
trace!(target: "state-trace", "{:04x}: ChildExists({}) {}={:?}",
self.id,
HexDisplay::from(&storage_key.as_ref()),
HexDisplay::from(&key),
result,
);
result
}
fn place_storage(&mut self, key: Vec<u8>, value: Option<Vec<u8>>) {
trace!(target: "state-trace", "{:04x}: Put {}={:?}",
self.id,
HexDisplay::from(&key),
value.as_ref().map(HexDisplay::from)
);
let _guard = panic_handler::AbortGuard::force_abort();
if is_child_storage_key(&key) {
warn!(target: "trie", "Refuse to directly set child storage key");
return;
}
self.mark_dirty();
self.overlay.set_storage(key, value);
}
fn place_child_storage(
&mut self,
storage_key: ChildStorageKey,
key: Vec<u8>,
value: Option<Vec<u8>>,
) {
trace!(target: "state-trace", "{:04x}: PutChild({}) {}={:?}",
self.id,
HexDisplay::from(&storage_key.as_ref()),
HexDisplay::from(&key),
value.as_ref().map(HexDisplay::from)
);
let _guard = panic_handler::AbortGuard::force_abort();
self.mark_dirty();
self.overlay.set_child_storage(storage_key.into_owned(), key, value);
}
fn kill_child_storage(&mut self, storage_key: ChildStorageKey) {
trace!(target: "state-trace", "{:04x}: KillChild({})",
self.id,
HexDisplay::from(&storage_key.as_ref()),
);
let _guard = panic_handler::AbortGuard::force_abort();
self.mark_dirty();
self.overlay.clear_child_storage(storage_key.as_ref());
self.backend.for_keys_in_child_storage(storage_key.as_ref(), |key| {
self.overlay.set_child_storage(storage_key.as_ref().to_vec(), key.to_vec(), None);
});
}
fn clear_prefix(&mut self, prefix: &[u8]) {
trace!(target: "state-trace", "{:04x}: ClearPrefix {}",
self.id,
HexDisplay::from(&prefix),
);
let _guard = panic_handler::AbortGuard::force_abort();
if is_child_storage_key(prefix) {
warn!(target: "trie", "Refuse to directly clear prefix that is part of child storage key");
return;
}
self.mark_dirty();
self.overlay.clear_prefix(prefix);
self.backend.for_keys_with_prefix(prefix, |key| {
self.overlay.set_storage(key.to_vec(), None);
});
}
fn clear_child_prefix(&mut self, storage_key: ChildStorageKey, prefix: &[u8]) {
trace!(target: "state-trace", "{:04x}: ClearChildPrefix({}) {}",
self.id,
HexDisplay::from(&storage_key.as_ref()),
HexDisplay::from(&prefix),
);
let _guard = panic_handler::AbortGuard::force_abort();
self.mark_dirty();
self.overlay.clear_child_prefix(storage_key.as_ref(), prefix);
self.backend.for_child_keys_with_prefix(storage_key.as_ref(), prefix, |key| {
self.overlay.set_child_storage(storage_key.as_ref().to_vec(), key.to_vec(), None);
});
}
fn chain_id(&self) -> u64 {
42
}
fn storage_root(&mut self) -> H256 {
let _guard = panic_handler::AbortGuard::force_abort();
if let Some((_, ref root)) = self.storage_transaction {
trace!(target: "state-trace", "{:04x}: Root (cached) {}",
self.id,
HexDisplay::from(&root.as_ref()),
);
return root.clone();
}
let child_storage_keys =
self.overlay.prospective.children.keys()
.chain(self.overlay.committed.children.keys());
let child_delta_iter = child_storage_keys.map(|storage_key|
(storage_key.clone(), self.overlay.committed.children.get(storage_key)
.into_iter()
.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))
.chain(self.overlay.prospective.children.get(storage_key)
.into_iter()
.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.value.clone()))))));
// compute and memoize
let delta = self.overlay.committed.top.iter().map(|(k, v)| (k.clone(), v.value.clone()))
.chain(self.overlay.prospective.top.iter().map(|(k, v)| (k.clone(), v.value.clone())));
let (root, transaction) = self.backend.full_storage_root(delta, child_delta_iter);
self.storage_transaction = Some((transaction, root));
trace!(target: "state-trace", "{:04x}: Root {}",
self.id,
HexDisplay::from(&root.as_ref()),
);
root
}
fn child_storage_root(&mut self, storage_key: ChildStorageKey) -> Vec<u8> {
let _guard = panic_handler::AbortGuard::force_abort();
if self.storage_transaction.is_some() {
let root = self
.storage(storage_key.as_ref())
.unwrap_or(
default_child_trie_root::<Layout<H>>(storage_key.as_ref())
);
trace!(target: "state-trace", "{:04x}: ChildRoot({}) (cached) {}",
self.id,
HexDisplay::from(&storage_key.as_ref()),
HexDisplay::from(&root.as_ref()),
);
root
} else {
let storage_key = storage_key.as_ref();
let (root, is_empty, _) = {
let delta = self.overlay.committed.children.get(storage_key)
.into_iter()
.flat_map(|map| map.clone().into_iter().map(|(k, v)| (k, v.value)))
.chain(self.overlay.prospective.children.get(storage_key)
.into_iter()
.flat_map(|map| map.clone().into_iter().map(|(k, v)| (k, v.value))));
self.backend.child_storage_root(storage_key, delta)
};
if is_empty {
self.overlay.set_storage(storage_key.into(), None);
} else {
self.overlay.set_storage(storage_key.into(), Some(root.clone()));
}
trace!(target: "state-trace", "{:04x}: ChildRoot({}) {}",
self.id,
HexDisplay::from(&storage_key.as_ref()),
HexDisplay::from(&root.as_ref()),
);
root
}
}
fn storage_changes_root(&mut self, parent_hash: H256) -> Result<Option<H256>, ()> {
let _guard = panic_handler::AbortGuard::force_abort();
self.changes_trie_transaction = build_changes_trie::<_, T, H, N>(
self.backend,
self.changes_trie_storage.clone(),
self.overlay,
parent_hash,
)?;
let result = Ok(self.changes_trie_transaction.as_ref().map(|(_, root, _)| root.clone()));
trace!(target: "state-trace", "{:04x}: ChangesRoot({}) {:?}",
self.id,
HexDisplay::from(&parent_hash.as_ref()),
result,
);
result
}
}
impl<'a, H, B, T, N> externalities::ExtensionStore for Ext<'a, H, N, B, T>
where
H: Hasher<Out=H256>,
B: 'a + Backend<H>,
T: 'a + ChangesTrieStorage<H, N>,
N: crate::changes_trie::BlockNumber,
{
fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any> {
self.extensions.as_mut().and_then(|exts| exts.get_mut(type_id))
}
}
#[cfg(test)]
mod tests {
use super::*;
use hex_literal::hex;
use codec::Encode;
use primitives::{Blake2Hasher, storage::well_known_keys::EXTRINSIC_INDEX};
use crate::{
changes_trie::{
Configuration as ChangesTrieConfiguration,
InMemoryStorage as InMemoryChangesTrieStorage,
}, backend::InMemory, overlayed_changes::OverlayedValue,
};
type TestBackend = InMemory<Blake2Hasher>;
type TestChangesTrieStorage = InMemoryChangesTrieStorage<Blake2Hasher, u64>;
type TestExt<'a> = Ext<'a, Blake2Hasher, u64, TestBackend, TestChangesTrieStorage>;
fn prepare_overlay_with_changes() -> OverlayedChanges {
OverlayedChanges {
prospective: vec![
(EXTRINSIC_INDEX.to_vec(), OverlayedValue {
value: Some(3u32.encode()),
extrinsics: Some(vec![1].into_iter().collect())
}),
(vec![1], OverlayedValue {
value: Some(vec![100].into_iter().collect()),
extrinsics: Some(vec![1].into_iter().collect())
}),
].into_iter().collect(),
committed: Default::default(),
changes_trie_config: Some(ChangesTrieConfiguration {
digest_interval: 0,
digest_levels: 0,
}),
}
}
#[test]
fn storage_changes_root_is_none_when_storage_is_not_provided() {
let mut overlay = prepare_overlay_with_changes();
let backend = TestBackend::default();
let mut ext = TestExt::new(&mut overlay, &backend, None, None);
assert_eq!(ext.storage_changes_root(Default::default()).unwrap(), None);
}
#[test]
fn storage_changes_root_is_none_when_extrinsic_changes_are_none() {
let mut overlay = prepare_overlay_with_changes();
overlay.changes_trie_config = None;
let storage = TestChangesTrieStorage::with_blocks(vec![(100, Default::default())]);
let backend = TestBackend::default();
let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None);
assert_eq!(ext.storage_changes_root(Default::default()).unwrap(), None);
}
#[test]
fn storage_changes_root_is_some_when_extrinsic_changes_are_non_empty() {
let mut overlay = prepare_overlay_with_changes();
let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
let backend = TestBackend::default();
let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None);
assert_eq!(
ext.storage_changes_root(Default::default()).unwrap(),
Some(hex!("bb0c2ef6e1d36d5490f9766cfcc7dfe2a6ca804504c3bb206053890d6dd02376").into()),
);
}
#[test]
fn storage_changes_root_is_some_when_extrinsic_changes_are_empty() {
let mut overlay = prepare_overlay_with_changes();
overlay.prospective.top.get_mut(&vec![1]).unwrap().value = None;
let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
let backend = TestBackend::default();
let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None);
assert_eq!(
ext.storage_changes_root(Default::default()).unwrap(),
Some(hex!("96f5aae4690e7302737b6f9b7f8567d5bbb9eac1c315f80101235a92d9ec27f4").into()),
);
}
}
substrate/primitives/state-machine/src/lib.rs
+1128
View File
File diff suppressed because it is too large Load Diff
substrate/primitives/state-machine/src/overlayed_changes.rs
+546
View File
@@ -0,0 +1,546 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! The overlayed changes to state.
#[cfg(test)]
use std::iter::FromIterator;
use std::collections::{HashMap, BTreeSet};
use codec::Decode;
use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig};
use primitives::storage::well_known_keys::EXTRINSIC_INDEX;
/// The overlayed changes to state to be queried on top of the backend.
///
/// A transaction shares all prospective changes within an inner overlay
/// that can be cleared.
#[derive(Debug, Default, Clone)]
pub struct OverlayedChanges {
/// Changes that are not yet committed.
pub(crate) prospective: OverlayedChangeSet,
/// Committed changes.
pub(crate) committed: OverlayedChangeSet,
/// Changes trie configuration. None by default, but could be installed by the
/// runtime if it supports change tries.
pub(crate) changes_trie_config: Option<ChangesTrieConfig>,
}
/// The storage value, used inside OverlayedChanges.
#[derive(Debug, Default, Clone)]
#[cfg_attr(test, derive(PartialEq))]
pub struct OverlayedValue {
/// Current value. None if value has been deleted.
pub value: Option<Vec<u8>>,
/// The set of extinsic indices where the values has been changed.
/// Is filled only if runtime has announced changes trie support.
pub extrinsics: Option<BTreeSet<u32>>,
}
/// Prospective or committed overlayed change set.
#[derive(Debug, Default, Clone)]
#[cfg_attr(test, derive(PartialEq))]
pub struct OverlayedChangeSet {
/// Top level storage changes.
pub top: HashMap<Vec<u8>, OverlayedValue>,
/// Child storage changes.
pub children: HashMap<Vec<u8>, HashMap<Vec<u8>, OverlayedValue>>,
}
#[cfg(test)]
impl FromIterator<(Vec<u8>, OverlayedValue)> for OverlayedChangeSet {
fn from_iter<T: IntoIterator<Item = (Vec<u8>, OverlayedValue)>>(iter: T) -> Self {
Self {
top: iter.into_iter().collect(),
children: Default::default(),
}
}
}
impl OverlayedChangeSet {
/// Whether the change set is empty.
pub fn is_empty(&self) -> bool {
self.top.is_empty() && self.children.is_empty()
}
/// Clear the change set.
pub fn clear(&mut self) {
self.top.clear();
self.children.clear();
}
}
impl OverlayedChanges {
/// Whether the overlayed changes are empty.
pub fn is_empty(&self) -> bool {
self.prospective.is_empty() && self.committed.is_empty()
}
/// Sets the changes trie configuration.
///
/// Returns false if configuration has been set already and we now trying
/// to install different configuration. This isn't supported now.
pub(crate) fn set_changes_trie_config(&mut self, config: ChangesTrieConfig) -> bool {
if let Some(ref old_config) = self.changes_trie_config {
// we do not support changes trie configuration' change now
if *old_config != config {
return false;
}
}
self.changes_trie_config = Some(config);
true
}
/// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered
/// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose
/// value has been set.
pub fn storage(&self, key: &[u8]) -> Option<Option<&[u8]>> {
self.prospective.top.get(key)
.or_else(|| self.committed.top.get(key))
.map(|x| x.value.as_ref().map(AsRef::as_ref))
}
/// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered
/// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose
/// value has been set.
pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option<Option<&[u8]>> {
if let Some(map) = self.prospective.children.get(storage_key) {
if let Some(val) = map.get(key) {
return Some(val.value.as_ref().map(AsRef::as_ref));
}
}
if let Some(map) = self.committed.children.get(storage_key) {
if let Some(val) = map.get(key) {
return Some(val.value.as_ref().map(AsRef::as_ref));
}
}
None
}
/// Inserts the given key-value pair into the prospective change set.
///
/// `None` can be used to delete a value specified by the given key.
pub(crate) fn set_storage(&mut self, key: Vec<u8>, val: Option<Vec<u8>>) {
let extrinsic_index = self.extrinsic_index();
let entry = self.prospective.top.entry(key).or_default();
entry.value = val;
if let Some(extrinsic) = extrinsic_index {
entry.extrinsics.get_or_insert_with(Default::default)
.insert(extrinsic);
}
}
/// Inserts the given key-value pair into the prospective child change set.
///
/// `None` can be used to delete a value specified by the given key.
pub(crate) fn set_child_storage(&mut self, storage_key: Vec<u8>, key: Vec<u8>, val: Option<Vec<u8>>) {
let extrinsic_index = self.extrinsic_index();
let map_entry = self.prospective.children.entry(storage_key).or_default();
let entry = map_entry.entry(key).or_default();
entry.value = val;
if let Some(extrinsic) = extrinsic_index {
entry.extrinsics.get_or_insert_with(Default::default)
.insert(extrinsic);
}
}
/// Clear child storage of given storage key.
///
/// NOTE that this doesn't take place immediately but written into the prospective
/// change set, and still can be reverted by [`discard_prospective`].
///
/// [`discard_prospective`]: #method.discard_prospective
pub(crate) fn clear_child_storage(&mut self, storage_key: &[u8]) {
let extrinsic_index = self.extrinsic_index();
let map_entry = self.prospective.children.entry(storage_key.to_vec()).or_default();
map_entry.values_mut().for_each(|e| {
if let Some(extrinsic) = extrinsic_index {
e.extrinsics.get_or_insert_with(Default::default)
.insert(extrinsic);
}
e.value = None;
});
if let Some(committed_map) = self.committed.children.get(storage_key) {
for (key, value) in committed_map.iter() {
if !map_entry.contains_key(key) {
map_entry.insert(key.clone(), OverlayedValue {
value: None,
extrinsics: extrinsic_index.map(|i| {
let mut e = value.extrinsics.clone()
.unwrap_or_else(|| BTreeSet::default());
e.insert(i);
e
}),
});
}
}
}
}
/// Removes all key-value pairs which keys share the given prefix.
///
/// NOTE that this doesn't take place immediately but written into the prospective
/// change set, and still can be reverted by [`discard_prospective`].
///
/// [`discard_prospective`]: #method.discard_prospective
pub(crate) fn clear_prefix(&mut self, prefix: &[u8]) {
let extrinsic_index = self.extrinsic_index();
// Iterate over all prospective and mark all keys that share
// the given prefix as removed (None).
for (key, entry) in self.prospective.top.iter_mut() {
if key.starts_with(prefix) {
entry.value = None;
if let Some(extrinsic) = extrinsic_index {
entry.extrinsics.get_or_insert_with(Default::default)
.insert(extrinsic);
}
}
}
// Then do the same with keys from commited changes.
// NOTE that we are making changes in the prospective change set.
for key in self.committed.top.keys() {
if key.starts_with(prefix) {
let entry = self.prospective.top.entry(key.clone()).or_default();
entry.value = None;
if let Some(extrinsic) = extrinsic_index {
entry.extrinsics.get_or_insert_with(Default::default)
.insert(extrinsic);
}
}
}
}
pub(crate) fn clear_child_prefix(&mut self, storage_key: &[u8], prefix: &[u8]) {
let extrinsic_index = self.extrinsic_index();
let map_entry = self.prospective.children.entry(storage_key.to_vec()).or_default();
for (key, entry) in map_entry.iter_mut() {
if key.starts_with(prefix) {
entry.value = None;
if let Some(extrinsic) = extrinsic_index {
entry.extrinsics.get_or_insert_with(Default::default)
.insert(extrinsic);
}
}
}
if let Some(child_committed) = self.committed.children.get(storage_key) {
// Then do the same with keys from commited changes.
// NOTE that we are making changes in the prospective change set.
for key in child_committed.keys() {
if key.starts_with(prefix) {
let entry = map_entry.entry(key.clone()).or_default();
entry.value = None;
if let Some(extrinsic) = extrinsic_index {
entry.extrinsics.get_or_insert_with(Default::default)
.insert(extrinsic);
}
}
}
}
}
/// Discard prospective changes to state.
pub fn discard_prospective(&mut self) {
self.prospective.clear();
}
/// Commit prospective changes to state.
pub fn commit_prospective(&mut self) {
if self.committed.is_empty() {
::std::mem::swap(&mut self.prospective, &mut self.committed);
} else {
for (key, val) in self.prospective.top.drain() {
let entry = self.committed.top.entry(key).or_default();
entry.value = val.value;
if let Some(prospective_extrinsics) = val.extrinsics {
entry.extrinsics.get_or_insert_with(Default::default)
.extend(prospective_extrinsics);
}
}
for (storage_key, mut map) in self.prospective.children.drain() {
let map_dest = self.committed.children.entry(storage_key).or_default();
for (key, val) in map.drain() {
let entry = map_dest.entry(key).or_default();
entry.value = val.value;
if let Some(prospective_extrinsics) = val.extrinsics {
entry.extrinsics.get_or_insert_with(Default::default)
.extend(prospective_extrinsics);
}
}
}
}
}
/// Consume `OverlayedChanges` and take committed set.
///
/// Panics:
/// Will panic if there are any uncommitted prospective changes.
pub fn into_committed(self) -> (
impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
impl Iterator<Item=(Vec<u8>, impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)>)>,
){
assert!(self.prospective.is_empty());
(self.committed.top.into_iter().map(|(k, v)| (k, v.value)),
self.committed.children.into_iter()
.map(|(sk, v)| (sk, v.into_iter().map(|(k, v)| (k, v.value)))))
}
/// Inserts storage entry responsible for current extrinsic index.
#[cfg(test)]
pub(crate) fn set_extrinsic_index(&mut self, extrinsic_index: u32) {
use codec::Encode;
self.prospective.top.insert(EXTRINSIC_INDEX.to_vec(), OverlayedValue {
value: Some(extrinsic_index.encode()),
extrinsics: None,
});
}
/// Returns current extrinsic index to use in changes trie construction.
/// None is returned if it is not set or changes trie config is not set.
/// Persistent value (from the backend) can be ignored because runtime must
/// set this index before first and unset after last extrinsic is executied.
/// Changes that are made outside of extrinsics, are marked with
/// `NO_EXTRINSIC_INDEX` index.
fn extrinsic_index(&self) -> Option<u32> {
match self.changes_trie_config.is_some() {
true => Some(
self.storage(EXTRINSIC_INDEX)
.and_then(|idx| idx.and_then(|idx| Decode::decode(&mut &*idx).ok()))
.unwrap_or(NO_EXTRINSIC_INDEX)),
false => None,
}
}
}
#[cfg(test)]
impl From<Option<Vec<u8>>> for OverlayedValue {
fn from(value: Option<Vec<u8>>) -> OverlayedValue {
OverlayedValue { value, ..Default::default() }
}
}
#[cfg(test)]
mod tests {
use hex_literal::hex;
use primitives::{
Blake2Hasher, H256, traits::Externalities, storage::well_known_keys::EXTRINSIC_INDEX,
};
use crate::backend::InMemory;
use crate::changes_trie::InMemoryStorage as InMemoryChangesTrieStorage;
use crate::ext::Ext;
use super::*;
fn strip_extrinsic_index(map: &HashMap<Vec<u8>, OverlayedValue>)
-> HashMap<Vec<u8>, OverlayedValue>
{
let mut clone = map.clone();
clone.remove(&EXTRINSIC_INDEX.to_vec());
clone
}
#[test]
fn overlayed_storage_works() {
let mut overlayed = OverlayedChanges::default();
let key = vec![42, 69, 169, 142];
assert!(overlayed.storage(&key).is_none());
overlayed.set_storage(key.clone(), Some(vec![1, 2, 3]));
assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..]));
overlayed.commit_prospective();
assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..]));
overlayed.set_storage(key.clone(), Some(vec![]));
assert_eq!(overlayed.storage(&key).unwrap(), Some(&[][..]));
overlayed.set_storage(key.clone(), None);
assert!(overlayed.storage(&key).unwrap().is_none());
overlayed.discard_prospective();
assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..]));
overlayed.set_storage(key.clone(), None);
overlayed.commit_prospective();
assert!(overlayed.storage(&key).unwrap().is_none());
}
#[test]
fn overlayed_storage_root_works() {
let initial: HashMap<_, _> = vec![
(b"doe".to_vec(), b"reindeer".to_vec()),
(b"dog".to_vec(), b"puppyXXX".to_vec()),
(b"dogglesworth".to_vec(), b"catXXX".to_vec()),
(b"doug".to_vec(), b"notadog".to_vec()),
].into_iter().collect();
let backend = InMemory::<Blake2Hasher>::from(initial);
let mut overlay = OverlayedChanges {
committed: vec![
(b"dog".to_vec(), Some(b"puppy".to_vec()).into()),
(b"dogglesworth".to_vec(), Some(b"catYYY".to_vec()).into()),
(b"doug".to_vec(), Some(vec![]).into()),
].into_iter().collect(),
prospective: vec![
(b"dogglesworth".to_vec(), Some(b"cat".to_vec()).into()),
(b"doug".to_vec(), None.into()),
].into_iter().collect(),
..Default::default()
};
let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
let mut ext = Ext::new(
&mut overlay,
&backend,
Some(&changes_trie_storage),
None,
);
const ROOT: [u8; 32] = hex!("39245109cef3758c2eed2ccba8d9b370a917850af3824bc8348d505df2c298fa");
assert_eq!(ext.storage_root(), H256::from(ROOT));
}
#[test]
fn changes_trie_configuration_is_saved() {
let mut overlay = OverlayedChanges::default();
assert!(overlay.changes_trie_config.is_none());
assert_eq!(
overlay.set_changes_trie_config(
ChangesTrieConfig { digest_interval: 4, digest_levels: 1, },
),
true,
);
assert!(overlay.changes_trie_config.is_some());
}
#[test]
fn changes_trie_configuration_is_saved_twice() {
let mut overlay = OverlayedChanges::default();
assert!(overlay.changes_trie_config.is_none());
assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
digest_interval: 4, digest_levels: 1,
}), true);
overlay.set_extrinsic_index(0);
overlay.set_storage(vec![1], Some(vec![2]));
assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
digest_interval: 4, digest_levels: 1,
}), true);
assert_eq!(
strip_extrinsic_index(&overlay.prospective.top),
vec![
(vec![1], OverlayedValue { value: Some(vec![2]),
extrinsics: Some(vec![0].into_iter().collect()) }),
].into_iter().collect(),
);
}
#[test]
fn panics_when_trying_to_save_different_changes_trie_configuration() {
let mut overlay = OverlayedChanges::default();
assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
digest_interval: 4, digest_levels: 1,
}), true);
assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
digest_interval: 2, digest_levels: 1,
}), false);
}
#[test]
fn extrinsic_changes_are_collected() {
let mut overlay = OverlayedChanges::default();
let _ = overlay.set_changes_trie_config(ChangesTrieConfig {
digest_interval: 4, digest_levels: 1,
});
overlay.set_storage(vec![100], Some(vec![101]));
overlay.set_extrinsic_index(0);
overlay.set_storage(vec![1], Some(vec![2]));
overlay.set_extrinsic_index(1);
overlay.set_storage(vec![3], Some(vec![4]));
overlay.set_extrinsic_index(2);
overlay.set_storage(vec![1], Some(vec![6]));
assert_eq!(strip_extrinsic_index(&overlay.prospective.top),
vec![
(vec![1], OverlayedValue { value: Some(vec![6]),
extrinsics: Some(vec![0, 2].into_iter().collect()) }),
(vec![3], OverlayedValue { value: Some(vec![4]),
extrinsics: Some(vec![1].into_iter().collect()) }),
(vec![100], OverlayedValue { value: Some(vec![101]),
extrinsics: Some(vec![NO_EXTRINSIC_INDEX].into_iter().collect()) }),
].into_iter().collect());
overlay.commit_prospective();
overlay.set_extrinsic_index(3);
overlay.set_storage(vec![3], Some(vec![7]));
overlay.set_extrinsic_index(4);
overlay.set_storage(vec![1], Some(vec![8]));
assert_eq!(strip_extrinsic_index(&overlay.committed.top),
vec![
(vec![1], OverlayedValue { value: Some(vec![6]),
extrinsics: Some(vec![0, 2].into_iter().collect()) }),
(vec![3], OverlayedValue { value: Some(vec![4]),
extrinsics: Some(vec![1].into_iter().collect()) }),
(vec![100], OverlayedValue { value: Some(vec![101]),
extrinsics: Some(vec![NO_EXTRINSIC_INDEX].into_iter().collect()) }),
].into_iter().collect());
assert_eq!(strip_extrinsic_index(&overlay.prospective.top),
vec![
(vec![1], OverlayedValue { value: Some(vec![8]),
extrinsics: Some(vec![4].into_iter().collect()) }),
(vec![3], OverlayedValue { value: Some(vec![7]),
extrinsics: Some(vec![3].into_iter().collect()) }),
].into_iter().collect());
overlay.commit_prospective();
assert_eq!(strip_extrinsic_index(&overlay.committed.top),
vec![
(vec![1], OverlayedValue { value: Some(vec![8]),
extrinsics: Some(vec![0, 2, 4].into_iter().collect()) }),
(vec![3], OverlayedValue { value: Some(vec![7]),
extrinsics: Some(vec![1, 3].into_iter().collect()) }),
(vec![100], OverlayedValue { value: Some(vec![101]),
extrinsics: Some(vec![NO_EXTRINSIC_INDEX].into_iter().collect()) }),
].into_iter().collect());
assert_eq!(overlay.prospective,
Default::default());
}
}
substrate/primitives/state-machine/src/proving_backend.rs
+484
View File
@@ -0,0 +1,484 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Proving state machine backend.
use std::sync::Arc;
use parking_lot::RwLock;
use codec::{Decode, Encode};
use log::debug;
use hash_db::{Hasher, HashDB, EMPTY_PREFIX, Prefix};
use trie::{
MemoryDB, PrefixedMemoryDB, default_child_trie_root,
read_trie_value_with, read_child_trie_value_with, record_all_keys
};
pub use trie::Recorder;
pub use trie::trie_types::{Layout, TrieError};
use crate::trie_backend::TrieBackend;
use crate::trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage};
use crate::{Error, ExecutionError, Backend};
use std::collections::{HashMap, HashSet};
use crate::DBValue;
/// Patricia trie-based backend specialized in get value proofs.
pub struct ProvingBackendRecorder<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> {
pub(crate) backend: &'a TrieBackendEssence<S, H>,
pub(crate) proof_recorder: &'a mut Recorder<H::Out>,
}
/// A proof that some set of key-value pairs are included in the storage trie. The proof contains
/// the storage values so that the partial storage backend can be reconstructed by a verifier that
/// does not already have access to the key-value pairs.
///
/// The proof consists of the set of serialized nodes in the storage trie accessed when looking up
/// the keys covered by the proof. Verifying the proof requires constructing the partial trie from
/// the serialized nodes and performing the key lookups.
#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
pub struct StorageProof {
trie_nodes: Vec<Vec<u8>>,
}
impl StorageProof {
/// Constructs a storage proof from a subset of encoded trie nodes in a storage backend.
pub fn new(trie_nodes: Vec<Vec<u8>>) -> Self {
StorageProof { trie_nodes }
}
/// Returns a new empty proof.
///
/// An empty proof is capable of only proving trivial statements (ie. that an empty set of
/// key-value pairs exist in storage).
pub fn empty() -> Self {
StorageProof {
trie_nodes: Vec::new(),
}
}
/// Returns whether this is an empty proof.
pub fn is_empty(&self) -> bool {
self.trie_nodes.is_empty()
}
/// Create an iterator over trie nodes constructed from the proof. The nodes are not guaranteed
/// to be traversed in any particular order.
pub fn iter_nodes(self) -> StorageProofNodeIterator {
StorageProofNodeIterator::new(self)
}
}
/// An iterator over trie nodes constructed from a storage proof. The nodes are not guaranteed to
/// be traversed in any particular order.
pub struct StorageProofNodeIterator {
inner: <Vec<Vec<u8>> as IntoIterator>::IntoIter,
}
impl StorageProofNodeIterator {
fn new(proof: StorageProof) -> Self {
StorageProofNodeIterator {
inner: proof.trie_nodes.into_iter(),
}
}
}
impl Iterator for StorageProofNodeIterator {
type Item = Vec<u8>;
fn next(&mut self) -> Option<Self::Item> {
self.inner.next()
}
}
/// Merges multiple storage proofs covering potentially different sets of keys into one proof
/// covering all keys. The merged proof output may be smaller than the aggregate size of the input
/// proofs due to deduplication of trie nodes.
pub fn merge_storage_proofs<I>(proofs: I) -> StorageProof
where I: IntoIterator<Item=StorageProof>
{
let trie_nodes = proofs.into_iter()
.flat_map(|proof| proof.iter_nodes())
.collect::<HashSet<_>>()
.into_iter()
.collect();
StorageProof { trie_nodes }
}
impl<'a, S, H> ProvingBackendRecorder<'a, S, H>
where
S: TrieBackendStorage<H>,
H: Hasher,
{
/// Produce proof for a key query.
pub fn storage(&mut self, key: &[u8]) -> Result<Option<Vec<u8>>, String> {
let mut read_overlay = S::Overlay::default();
let eph = Ephemeral::new(
self.backend.backend_storage(),
&mut read_overlay,
);
let map_e = |e| format!("Trie lookup error: {}", e);
read_trie_value_with::<Layout<H>, _, Ephemeral<S, H>>(
&eph,
self.backend.root(),
key,
&mut *self.proof_recorder
).map_err(map_e)
}
/// Produce proof for a child key query.
pub fn child_storage(
&mut self,
storage_key: &[u8],
key: &[u8]
) -> Result<Option<Vec<u8>>, String> {
let root = self.storage(storage_key)?
.unwrap_or(default_child_trie_root::<Layout<H>>(storage_key));
let mut read_overlay = S::Overlay::default();
let eph = Ephemeral::new(
self.backend.backend_storage(),
&mut read_overlay,
);
let map_e = |e| format!("Trie lookup error: {}", e);
read_child_trie_value_with::<Layout<H>, _, _>(
storage_key,
&eph,
&root,
key,
&mut *self.proof_recorder
).map_err(map_e)
}
/// Produce proof for the whole backend.
pub fn record_all_keys(&mut self) {
let mut read_overlay = S::Overlay::default();
let eph = Ephemeral::new(
self.backend.backend_storage(),
&mut read_overlay,
);
let mut iter = move || -> Result<(), Box<TrieError<H::Out>>> {
let root = self.backend.root();
record_all_keys::<Layout<H>, _>(&eph, root, &mut *self.proof_recorder)
};
if let Err(e) = iter() {
debug!(target: "trie", "Error while recording all keys: {}", e);
}
}
}
/// Global proof recorder, act as a layer over a hash db for recording queried
/// data.
pub type ProofRecorder<H> = Arc<RwLock<HashMap<<H as Hasher>::Out, Option<DBValue>>>>;
/// Patricia trie-based backend which also tracks all touched storage trie values.
/// These can be sent to remote node and used as a proof of execution.
pub struct ProvingBackend<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> (
TrieBackend<ProofRecorderBackend<'a, S, H>, H>,
);
/// Trie backend storage with its proof recorder.
pub struct ProofRecorderBackend<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> {
backend: &'a S,
proof_recorder: ProofRecorder<H>,
}
impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> ProvingBackend<'a, S, H> {
/// Create new proving backend.
pub fn new(backend: &'a TrieBackend<S, H>) -> Self {
let proof_recorder = Default::default();
Self::new_with_recorder(backend, proof_recorder)
}
/// Create new proving backend with the given recorder.
pub fn new_with_recorder(
backend: &'a TrieBackend<S, H>,
proof_recorder: ProofRecorder<H>,
) -> Self {
let essence = backend.essence();
let root = essence.root().clone();
let recorder = ProofRecorderBackend {
backend: essence.backend_storage(),
proof_recorder: proof_recorder,
};
ProvingBackend(TrieBackend::new(recorder, root))
}
/// Extracting the gathered unordered proof.
pub fn extract_proof(&self) -> StorageProof {
let trie_nodes = self.0.essence().backend_storage().proof_recorder
.read()
.iter()
.filter_map(|(_k, v)| v.as_ref().map(|v| v.to_vec()))
.collect();
StorageProof::new(trie_nodes)
}
}
impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> TrieBackendStorage<H> for ProofRecorderBackend<'a, S, H> {
type Overlay = S::Overlay;
fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
if let Some(v) = self.proof_recorder.read().get(key) {
return Ok(v.clone());
}
let backend_value = self.backend.get(key, prefix)?;
self.proof_recorder.write().insert(key.clone(), backend_value.clone());
Ok(backend_value)
}
}
impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> std::fmt::Debug for ProvingBackend<'a, S, H> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "ProvingBackend")
}
}
impl<'a, S, H> Backend<H> for ProvingBackend<'a, S, H>
where
S: 'a + TrieBackendStorage<H>,
H: 'a + Hasher,
H::Out: Ord,
{
type Error = String;
type Transaction = S::Overlay;
type TrieBackendStorage = PrefixedMemoryDB<H>;
fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
self.0.storage(key)
}
fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
self.0.child_storage(storage_key, key)
}
fn for_keys_in_child_storage<F: FnMut(&[u8])>(&self, storage_key: &[u8], f: F) {
self.0.for_keys_in_child_storage(storage_key, f)
}
fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
self.0.for_keys_with_prefix(prefix, f)
}
fn for_key_values_with_prefix<F: FnMut(&[u8], &[u8])>(&self, prefix: &[u8], f: F) {
self.0.for_key_values_with_prefix(prefix, f)
}
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(&self, storage_key: &[u8], prefix: &[u8], f: F) {
self.0.for_child_keys_with_prefix(storage_key, prefix, f)
}
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
self.0.pairs()
}
fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
self.0.keys(prefix)
}
fn child_keys(&self, child_storage_key: &[u8], prefix: &[u8]) -> Vec<Vec<u8>> {
self.0.child_keys(child_storage_key, prefix)
}
fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
{
self.0.storage_root(delta)
}
fn child_storage_root<I>(&self, storage_key: &[u8], delta: I) -> (Vec<u8>, bool, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
H::Out: Ord
{
self.0.child_storage_root(storage_key, delta)
}
}
/// Create proof check backend.
pub fn create_proof_check_backend<H>(
root: H::Out,
proof: StorageProof,
) -> Result<TrieBackend<MemoryDB<H>, H>, Box<dyn Error>>
where
H: Hasher,
{
let db = create_proof_check_backend_storage(proof);
if db.contains(&root, EMPTY_PREFIX) {
Ok(TrieBackend::new(db, root))
} else {
Err(Box::new(ExecutionError::InvalidProof))
}
}
/// Create in-memory storage of proof check backend.
pub fn create_proof_check_backend_storage<H>(
proof: StorageProof,
) -> MemoryDB<H>
where
H: Hasher,
{
let mut db = MemoryDB::default();
for item in proof.iter_nodes() {
db.insert(EMPTY_PREFIX, &item);
}
db
}
#[cfg(test)]
mod tests {
use crate::backend::{InMemory};
use crate::trie_backend::tests::test_trie;
use super::*;
use primitives::{Blake2Hasher, storage::ChildStorageKey};
use crate::proving_backend::create_proof_check_backend;
fn test_proving<'a>(
trie_backend: &'a TrieBackend<PrefixedMemoryDB<Blake2Hasher>,Blake2Hasher>,
) -> ProvingBackend<'a, PrefixedMemoryDB<Blake2Hasher>, Blake2Hasher> {
ProvingBackend::new(trie_backend)
}
#[test]
fn proof_is_empty_until_value_is_read() {
let trie_backend = test_trie();
assert!(test_proving(&trie_backend).extract_proof().is_empty());
}
#[test]
fn proof_is_non_empty_after_value_is_read() {
let trie_backend = test_trie();
let backend = test_proving(&trie_backend);
assert_eq!(backend.storage(b"key").unwrap(), Some(b"value".to_vec()));
assert!(!backend.extract_proof().is_empty());
}
#[test]
fn proof_is_invalid_when_does_not_contains_root() {
use primitives::H256;
let result = create_proof_check_backend::<Blake2Hasher>(
H256::from_low_u64_be(1),
StorageProof::empty()
);
assert!(result.is_err());
}
#[test]
fn passes_throgh_backend_calls() {
let trie_backend = test_trie();
let proving_backend = test_proving(&trie_backend);
assert_eq!(trie_backend.storage(b"key").unwrap(), proving_backend.storage(b"key").unwrap());
assert_eq!(trie_backend.pairs(), proving_backend.pairs());
let (trie_root, mut trie_mdb) = trie_backend.storage_root(::std::iter::empty());
let (proving_root, mut proving_mdb) = proving_backend.storage_root(::std::iter::empty());
assert_eq!(trie_root, proving_root);
assert_eq!(trie_mdb.drain(), proving_mdb.drain());
}
#[test]
fn proof_recorded_and_checked() {
let contents = (0..64).map(|i| (None, vec![i], Some(vec![i]))).collect::<Vec<_>>();
let in_memory = InMemory::<Blake2Hasher>::default();
let mut in_memory = in_memory.update(contents);
let in_memory_root = in_memory.storage_root(::std::iter::empty()).0;
(0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i]));
let trie = in_memory.as_trie_backend().unwrap();
let trie_root = trie.storage_root(::std::iter::empty()).0;
assert_eq!(in_memory_root, trie_root);
(0..64).for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i]));
let proving = ProvingBackend::new(trie);
assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42]);
let proof = proving.extract_proof();
let proof_check = create_proof_check_backend::<Blake2Hasher>(in_memory_root.into(), proof).unwrap();
assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42]);
}
#[test]
fn proof_recorded_and_checked_with_child() {
let subtrie1 = ChildStorageKey::from_slice(b":child_storage:default:sub1").unwrap();
let subtrie2 = ChildStorageKey::from_slice(b":child_storage:default:sub2").unwrap();
let own1 = subtrie1.into_owned();
let own2 = subtrie2.into_owned();
let contents = (0..64).map(|i| (None, vec![i], Some(vec![i])))
.chain((28..65).map(|i| (Some(own1.clone()), vec![i], Some(vec![i]))))
.chain((10..15).map(|i| (Some(own2.clone()), vec![i], Some(vec![i]))))
.collect::<Vec<_>>();
let in_memory = InMemory::<Blake2Hasher>::default();
let mut in_memory = in_memory.update(contents);
let in_memory_root = in_memory.full_storage_root::<_, Vec<_>, _>(
::std::iter::empty(),
in_memory.child_storage_keys().map(|k|(k.to_vec(), Vec::new()))
).0;
(0..64).for_each(|i| assert_eq!(
in_memory.storage(&[i]).unwrap().unwrap(),
vec![i]
));
(28..65).for_each(|i| assert_eq!(
in_memory.child_storage(&own1[..], &[i]).unwrap().unwrap(),
vec![i]
));
(10..15).for_each(|i| assert_eq!(
in_memory.child_storage(&own2[..], &[i]).unwrap().unwrap(),
vec![i]
));
let trie = in_memory.as_trie_backend().unwrap();
let trie_root = trie.storage_root(::std::iter::empty()).0;
assert_eq!(in_memory_root, trie_root);
(0..64).for_each(|i| assert_eq!(
trie.storage(&[i]).unwrap().unwrap(),
vec![i]
));
let proving = ProvingBackend::new(trie);
assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42]);
let proof = proving.extract_proof();
let proof_check = create_proof_check_backend::<Blake2Hasher>(
in_memory_root.into(),
proof
).unwrap();
assert!(proof_check.storage(&[0]).is_err());
assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42]);
// note that it is include in root because proof close
assert_eq!(proof_check.storage(&[41]).unwrap().unwrap(), vec![41]);
assert_eq!(proof_check.storage(&[64]).unwrap(), None);
let proving = ProvingBackend::new(trie);
assert_eq!(proving.child_storage(&own1[..], &[64]), Ok(Some(vec![64])));
let proof = proving.extract_proof();
let proof_check = create_proof_check_backend::<Blake2Hasher>(
in_memory_root.into(),
proof
).unwrap();
assert_eq!(
proof_check.child_storage(&own1[..], &[64]).unwrap().unwrap(),
vec![64]
);
}
}
substrate/primitives/state-machine/src/testing.rs
+200
View File
@@ -0,0 +1,200 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Test implementation for Externalities.
use std::{collections::HashMap, any::{Any, TypeId}};
use hash_db::Hasher;
use crate::{
backend::{InMemory, Backend}, OverlayedChanges,
changes_trie::{
InMemoryStorage as ChangesTrieInMemoryStorage,
BlockNumber as ChangesTrieBlockNumber,
},
ext::Ext,
};
use primitives::{
storage::{
well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES, is_child_storage_key}
},
hash::H256, Blake2Hasher,
};
use codec::Encode;
use externalities::{Extensions, Extension};
type StorageTuple = (HashMap<Vec<u8>, Vec<u8>>, HashMap<Vec<u8>, HashMap<Vec<u8>, Vec<u8>>>);
/// Simple HashMap-based Externalities impl.
pub struct TestExternalities<H: Hasher<Out=H256>=Blake2Hasher, N: ChangesTrieBlockNumber=u64> {
overlay: OverlayedChanges,
backend: InMemory<H>,
changes_trie_storage: ChangesTrieInMemoryStorage<H, N>,
extensions: Extensions,
}
impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> TestExternalities<H, N> {
/// Get externalities implementation.
pub fn ext(&mut self) -> Ext<H, N, InMemory<H>, ChangesTrieInMemoryStorage<H, N>> {
Ext::new(
&mut self.overlay,
&self.backend,
Some(&self.changes_trie_storage),
Some(&mut self.extensions),
)
}
/// Create a new instance of `TestExternalities` with storage.
pub fn new(storage: StorageTuple) -> Self {
Self::new_with_code(&[], storage)
}
/// Create a new instance of `TestExternalities` with code and storage.
pub fn new_with_code(code: &[u8], mut storage: StorageTuple) -> Self {
let mut overlay = OverlayedChanges::default();
assert!(storage.0.keys().all(|key| !is_child_storage_key(key)));
assert!(storage.1.keys().all(|key| is_child_storage_key(key)));
super::set_changes_trie_config(
&mut overlay,
storage.0.get(&CHANGES_TRIE_CONFIG.to_vec()).cloned(),
false,
).expect("changes trie configuration is correct in test env; qed");
storage.0.insert(HEAP_PAGES.to_vec(), 8u64.encode());
storage.0.insert(CODE.to_vec(), code.to_vec());
let backend: HashMap<_, _> = storage.1.into_iter()
.map(|(keyspace, map)| (Some(keyspace), map))
.chain(Some((None, storage.0)).into_iter())
.collect();
TestExternalities {
overlay,
changes_trie_storage: ChangesTrieInMemoryStorage::new(),
backend: backend.into(),
extensions: Default::default(),
}
}
/// Insert key/value into backend
pub fn insert(&mut self, k: Vec<u8>, v: Vec<u8>) {
self.backend = self.backend.update(vec![(None, k, Some(v))]);
}
/// Registers the given extension for this instance.
pub fn register_extension<E: Any + Extension>(&mut self, ext: E) {
self.extensions.register(ext);
}
/// Get mutable reference to changes trie storage.
pub fn changes_trie_storage(&mut self) -> &mut ChangesTrieInMemoryStorage<H, N> {
&mut self.changes_trie_storage
}
/// Return a new backend with all pending value.
pub fn commit_all(&self) -> InMemory<H> {
let top = self.overlay.committed.top.clone().into_iter()
.chain(self.overlay.prospective.top.clone().into_iter())
.map(|(k, v)| (None, k, v.value));
let children = self.overlay.committed.children.clone().into_iter()
.chain(self.overlay.prospective.children.clone().into_iter())
.flat_map(|(keyspace, map)| {
map.into_iter()
.map(|(k, v)| (Some(keyspace.clone()), k, v.value))
.collect::<Vec<_>>()
});
self.backend.update(top.chain(children).collect())
}
/// Execute the given closure while `self` is set as externalities.
///
/// Returns the result of the given closure.
pub fn execute_with<R>(&mut self, execute: impl FnOnce() -> R) -> R {
let mut ext = self.ext();
externalities::set_and_run_with_externalities(&mut ext, execute)
}
}
impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> std::fmt::Debug for TestExternalities<H, N> {
fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
write!(f, "overlay: {:?}\nbackend: {:?}", self.overlay, self.backend.pairs())
}
}
impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> PartialEq for TestExternalities<H, N> {
/// This doesn't test if they are in the same state, only if they contains the
/// same data at this state
fn eq(&self, other: &TestExternalities<H, N>) -> bool {
self.commit_all().eq(&other.commit_all())
}
}
impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> Default for TestExternalities<H, N> {
fn default() -> Self { Self::new(Default::default()) }
}
impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> From<StorageTuple> for TestExternalities<H, N> {
fn from(storage: StorageTuple) -> Self {
Self::new(storage)
}
}
impl<H, N> externalities::ExtensionStore for TestExternalities<H, N> where
H: Hasher<Out=H256>,
N: ChangesTrieBlockNumber,
{
fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any> {
self.extensions.get_mut(type_id)
}
}
#[cfg(test)]
mod tests {
use super::*;
use primitives::traits::Externalities;
use hex_literal::hex;
#[test]
fn commit_should_work() {
let mut ext = TestExternalities::<Blake2Hasher, u64>::default();
let mut ext = ext.ext();
ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec());
ext.set_storage(b"dog".to_vec(), b"puppy".to_vec());
ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec());
const ROOT: [u8; 32] = hex!("2a340d3dfd52f5992c6b117e9e45f479e6da5afffafeb26ab619cf137a95aeb8");
assert_eq!(ext.storage_root(), H256::from(ROOT));
}
#[test]
fn set_and_retrieve_code() {
let mut ext = TestExternalities::<Blake2Hasher, u64>::default();
let mut ext = ext.ext();
let code = vec![1, 2, 3];
ext.set_storage(CODE.to_vec(), code.clone());
assert_eq!(&ext.storage(CODE).unwrap(), &code);
}
#[test]
fn check_send() {
fn assert_send<T: Send>() {}
assert_send::<TestExternalities::<Blake2Hasher, u64>>();
}
}
substrate/primitives/state-machine/src/trie_backend.rs
+298
View File
@@ -0,0 +1,298 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Trie-based state machine backend.
use log::{warn, debug};
use hash_db::Hasher;
use trie::{Trie, delta_trie_root, default_child_trie_root, child_delta_trie_root};
use trie::trie_types::{TrieDB, TrieError, Layout};
use crate::trie_backend_essence::{TrieBackendEssence, TrieBackendStorage, Ephemeral};
use crate::Backend;
/// Patricia trie-based backend. Transaction type is an overlay of changes to commit.
pub struct TrieBackend<S: TrieBackendStorage<H>, H: Hasher> {
essence: TrieBackendEssence<S, H>,
}
impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackend<S, H> {
/// Create new trie-based backend.
pub fn new(storage: S, root: H::Out) -> Self {
TrieBackend {
essence: TrieBackendEssence::new(storage, root),
}
}
/// Get backend essence reference.
pub fn essence(&self) -> &TrieBackendEssence<S, H> {
&self.essence
}
/// Get backend storage reference.
pub fn backend_storage(&self) -> &S {
self.essence.backend_storage()
}
/// Get trie root.
pub fn root(&self) -> &H::Out {
self.essence.root()
}
/// Consumes self and returns underlying storage.
pub fn into_storage(self) -> S {
self.essence.into_storage()
}
}
impl<S: TrieBackendStorage<H>, H: Hasher> std::fmt::Debug for TrieBackend<S, H> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "TrieBackend")
}
}
impl<S: TrieBackendStorage<H>, H: Hasher> Backend<H> for TrieBackend<S, H> where
H::Out: Ord,
{
type Error = String;
type Transaction = S::Overlay;
type TrieBackendStorage = S;
fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
self.essence.storage(key)
}
fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
self.essence.child_storage(storage_key, key)
}
fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
self.essence.for_keys_with_prefix(prefix, f)
}
fn for_key_values_with_prefix<F: FnMut(&[u8], &[u8])>(&self, prefix: &[u8], f: F) {
self.essence.for_key_values_with_prefix(prefix, f)
}
fn for_keys_in_child_storage<F: FnMut(&[u8])>(&self, storage_key: &[u8], f: F) {
self.essence.for_keys_in_child_storage(storage_key, f)
}
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(&self, storage_key: &[u8], prefix: &[u8], f: F) {
self.essence.for_child_keys_with_prefix(storage_key, prefix, f)
}
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
let mut read_overlay = S::Overlay::default();
let eph = Ephemeral::new(self.essence.backend_storage(), &mut read_overlay);
let collect_all = || -> Result<_, Box<TrieError<H::Out>>> {
let trie = TrieDB::<H>::new(&eph, self.essence.root())?;
let mut v = Vec::new();
for x in trie.iter()? {
let (key, value) = x?;
v.push((key.to_vec(), value.to_vec()));
}
Ok(v)
};
match collect_all() {
Ok(v) => v,
Err(e) => {
debug!(target: "trie", "Error extracting trie values: {}", e);
Vec::new()
}
}
}
fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
let mut read_overlay = S::Overlay::default();
let eph = Ephemeral::new(self.essence.backend_storage(), &mut read_overlay);
let collect_all = || -> Result<_, Box<TrieError<H::Out>>> {
let trie = TrieDB::<H>::new(&eph, self.essence.root())?;
let mut v = Vec::new();
for x in trie.iter()? {
let (key, _) = x?;
if key.starts_with(prefix) {
v.push(key.to_vec());
}
}
Ok(v)
};
collect_all().map_err(|e| debug!(target: "trie", "Error extracting trie keys: {}", e)).unwrap_or_default()
}
fn storage_root<I>(&self, delta: I) -> (H::Out, S::Overlay)
where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
{
let mut write_overlay = S::Overlay::default();
let mut root = *self.essence.root();
{
let mut eph = Ephemeral::new(
self.essence.backend_storage(),
&mut write_overlay,
);
match delta_trie_root::<Layout<H>, _, _, _, _>(&mut eph, root, delta) {
Ok(ret) => root = ret,
Err(e) => warn!(target: "trie", "Failed to write to trie: {}", e),
}
}
(root, write_overlay)
}
fn child_storage_root<I>(&self, storage_key: &[u8], delta: I) -> (Vec<u8>, bool, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
H::Out: Ord
{
let default_root = default_child_trie_root::<Layout<H>>(storage_key);
let mut write_overlay = S::Overlay::default();
let mut root = match self.storage(storage_key) {
Ok(value) => value.unwrap_or(default_root.clone()),
Err(e) => {
warn!(target: "trie", "Failed to read child storage root: {}", e);
default_root.clone()
},
};
{
let mut eph = Ephemeral::new(
self.essence.backend_storage(),
&mut write_overlay,
);
match child_delta_trie_root::<Layout<H>, _, _, _, _>(
storage_key,
&mut eph,
root.clone(),
delta
) {
Ok(ret) => root = ret,
Err(e) => warn!(target: "trie", "Failed to write to trie: {}", e),
}
}
let is_default = root == default_root;
(root, is_default, write_overlay)
}
fn as_trie_backend(&mut self) -> Option<&TrieBackend<Self::TrieBackendStorage, H>> {
Some(self)
}
}
#[cfg(test)]
pub mod tests {
use std::collections::HashSet;
use primitives::{Blake2Hasher, H256};
use codec::Encode;
use trie::{TrieMut, PrefixedMemoryDB, trie_types::TrieDBMut};
use super::*;
fn test_db() -> (PrefixedMemoryDB<Blake2Hasher>, H256) {
let mut root = H256::default();
let mut mdb = PrefixedMemoryDB::<Blake2Hasher>::default();
{
let mut trie = TrieDBMut::new(&mut mdb, &mut root);
trie.insert(b"value3", &[142]).expect("insert failed");
trie.insert(b"value4", &[124]).expect("insert failed");
};
{
let mut sub_root = Vec::new();
root.encode_to(&mut sub_root);
let mut trie = TrieDBMut::new(&mut mdb, &mut root);
trie.insert(b":child_storage:default:sub1", &sub_root).expect("insert failed");
trie.insert(b"key", b"value").expect("insert failed");
trie.insert(b"value1", &[42]).expect("insert failed");
trie.insert(b"value2", &[24]).expect("insert failed");
trie.insert(b":code", b"return 42").expect("insert failed");
for i in 128u8..255u8 {
trie.insert(&[i], &[i]).unwrap();
}
}
(mdb, root)
}
pub(crate) fn test_trie() -> TrieBackend<PrefixedMemoryDB<Blake2Hasher>, Blake2Hasher> {
let (mdb, root) = test_db();
TrieBackend::new(mdb, root)
}
#[test]
fn read_from_storage_returns_some() {
assert_eq!(test_trie().storage(b"key").unwrap(), Some(b"value".to_vec()));
}
#[test]
fn read_from_storage_returns_none() {
assert_eq!(test_trie().storage(b"non-existing-key").unwrap(), None);
}
#[test]
fn pairs_are_not_empty_on_non_empty_storage() {
assert!(!test_trie().pairs().is_empty());
}
#[test]
fn pairs_are_empty_on_empty_storage() {
assert!(TrieBackend::<PrefixedMemoryDB<Blake2Hasher>, Blake2Hasher>::new(
PrefixedMemoryDB::default(),
Default::default(),
).pairs().is_empty());
}
#[test]
fn storage_root_is_non_default() {
assert!(test_trie().storage_root(::std::iter::empty()).0 != H256::repeat_byte(0));
}
#[test]
fn storage_root_transaction_is_empty() {
assert!(test_trie().storage_root(::std::iter::empty()).1.drain().is_empty());
}
#[test]
fn storage_root_transaction_is_non_empty() {
let (new_root, mut tx) = test_trie().storage_root(vec![(b"new-key".to_vec(), Some(b"new-value".to_vec()))]);
assert!(!tx.drain().is_empty());
assert!(new_root != test_trie().storage_root(::std::iter::empty()).0);
}
#[test]
fn prefix_walking_works() {
let trie = test_trie();
let mut seen = HashSet::new();
trie.for_keys_with_prefix(b"value", |key| {
let for_first_time = seen.insert(key.to_vec());
assert!(for_first_time, "Seen key '{:?}' more than once", key);
});
let mut expected = HashSet::new();
expected.insert(b"value1".to_vec());
expected.insert(b"value2".to_vec());
assert_eq!(seen, expected);
}
}
substrate/primitives/state-machine/src/trie_backend_essence.rs
+341
View File
@@ -0,0 +1,341 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Trie-based state machine backend essence used to read values
//! from storage.
use std::ops::Deref;
use std::sync::Arc;
use log::{debug, warn};
use hash_db::{self, Hasher, EMPTY_PREFIX, Prefix};
use trie::{Trie, MemoryDB, PrefixedMemoryDB, DBValue,
default_child_trie_root, read_trie_value, read_child_trie_value,
for_keys_in_child_trie};
use trie::trie_types::{TrieDB, TrieError, Layout};
use crate::backend::Consolidate;
/// Patricia trie-based storage trait.
pub trait Storage<H: Hasher>: Send + Sync {
/// Get a trie node.
fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String>;
}
/// Patricia trie-based pairs storage essence.
pub struct TrieBackendEssence<S: TrieBackendStorage<H>, H: Hasher> {
storage: S,
root: H::Out,
}
impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> {
/// Create new trie-based backend.
pub fn new(storage: S, root: H::Out) -> Self {
TrieBackendEssence {
storage,
root,
}
}
/// Get backend storage reference.
pub fn backend_storage(&self) -> &S {
&self.storage
}
/// Get trie root.
pub fn root(&self) -> &H::Out {
&self.root
}
/// Consumes self and returns underlying storage.
pub fn into_storage(self) -> S {
self.storage
}
/// Get the value of storage at given key.
pub fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, String> {
let mut read_overlay = S::Overlay::default();
let eph = Ephemeral {
storage: &self.storage,
overlay: &mut read_overlay,
};
let map_e = |e| format!("Trie lookup error: {}", e);
read_trie_value::<Layout<H>, _>(&eph, &self.root, key).map_err(map_e)
}
/// Get the value of child storage at given key.
pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, String> {
let root = self.storage(storage_key)?
.unwrap_or(default_child_trie_root::<Layout<H>>(storage_key));
let mut read_overlay = S::Overlay::default();
let eph = Ephemeral {
storage: &self.storage,
overlay: &mut read_overlay,
};
let map_e = |e| format!("Trie lookup error: {}", e);
read_child_trie_value::<Layout<H>, _>(storage_key, &eph, &root, key).map_err(map_e)
}
/// Retrieve all entries keys of child storage and call `f` for each of those keys.
pub fn for_keys_in_child_storage<F: FnMut(&[u8])>(&self, storage_key: &[u8], f: F) {
let root = match self.storage(storage_key) {
Ok(v) => v.unwrap_or(default_child_trie_root::<Layout<H>>(storage_key)),
Err(e) => {
debug!(target: "trie", "Error while iterating child storage: {}", e);
return;
}
};
let mut read_overlay = S::Overlay::default();
let eph = Ephemeral {
storage: &self.storage,
overlay: &mut read_overlay,
};
if let Err(e) = for_keys_in_child_trie::<Layout<H>, _, Ephemeral<S, H>>(
storage_key,
&eph,
&root,
f,
) {
debug!(target: "trie", "Error while iterating child storage: {}", e);
}
}
/// Execute given closure for all keys starting with prefix.
pub fn for_child_keys_with_prefix<F: FnMut(&[u8])>(&self, storage_key: &[u8], prefix: &[u8], mut f: F) {
let root_vec = match self.storage(storage_key) {
Ok(v) => v.unwrap_or(default_child_trie_root::<Layout<H>>(storage_key)),
Err(e) => {
debug!(target: "trie", "Error while iterating child storage: {}", e);
return;
}
};
let mut root = H::Out::default();
root.as_mut().copy_from_slice(&root_vec);
self.keys_values_with_prefix_inner(&root, prefix, |k, _v| f(k))
}
/// Execute given closure for all keys starting with prefix.
pub fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], mut f: F) {
self.keys_values_with_prefix_inner(&self.root, prefix, |k, _v| f(k))
}
fn keys_values_with_prefix_inner<F: FnMut(&[u8], &[u8])>(
&self,
root: &H::Out,
prefix: &[u8],
mut f: F,
) {
let mut read_overlay = S::Overlay::default();
let eph = Ephemeral {
storage: &self.storage,
overlay: &mut read_overlay,
};
let mut iter = move || -> Result<(), Box<TrieError<H::Out>>> {
let trie = TrieDB::<H>::new(&eph, root)?;
let mut iter = trie.iter()?;
iter.seek(prefix)?;
for x in iter {
let (key, value) = x?;
if !key.starts_with(prefix) {
break;
}
f(&key, &value);
}
Ok(())
};
if let Err(e) = iter() {
debug!(target: "trie", "Error while iterating by prefix: {}", e);
}
}
/// Execute given closure for all key and values starting with prefix.
pub fn for_key_values_with_prefix<F: FnMut(&[u8], &[u8])>(&self, prefix: &[u8], f: F) {
self.keys_values_with_prefix_inner(&self.root, prefix, f)
}
}
pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> {
storage: &'a S,
overlay: &'a mut S::Overlay,
}
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: 'a + Hasher
> hash_db::AsPlainDB<H::Out, DBValue>
for Ephemeral<'a, S, H>
{
fn as_plain_db<'b>(&'b self) -> &'b (dyn hash_db::PlainDB<H::Out, DBValue> + 'b) { self }
fn as_plain_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::PlainDB<H::Out, DBValue> + 'b) { self }
}
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: 'a + Hasher
> hash_db::AsHashDB<H, DBValue>
for Ephemeral<'a, S, H>
{
fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB<H, DBValue> + 'b) { self }
fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB<H, DBValue> + 'b) { self }
}
impl<'a, S: TrieBackendStorage<H>, H: Hasher> Ephemeral<'a, S, H> {
pub fn new(storage: &'a S, overlay: &'a mut S::Overlay) -> Self {
Ephemeral {
storage,
overlay,
}
}
}
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: Hasher
> hash_db::PlainDB<H::Out, DBValue>
for Ephemeral<'a, S, H>
{
fn get(&self, key: &H::Out) -> Option<DBValue> {
if let Some(val) = hash_db::HashDB::get(self.overlay, key, EMPTY_PREFIX) {
Some(val)
} else {
match self.storage.get(&key, EMPTY_PREFIX) {
Ok(x) => x,
Err(e) => {
warn!(target: "trie", "Failed to read from DB: {}", e);
None
},
}
}
}
fn contains(&self, key: &H::Out) -> bool {
hash_db::HashDB::get(self, key, EMPTY_PREFIX).is_some()
}
fn emplace(&mut self, key: H::Out, value: DBValue) {
hash_db::HashDB::emplace(self.overlay, key, EMPTY_PREFIX, value)
}
fn remove(&mut self, key: &H::Out) {
hash_db::HashDB::remove(self.overlay, key, EMPTY_PREFIX)
}
}
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: Hasher
> hash_db::PlainDBRef<H::Out, DBValue>
for Ephemeral<'a, S, H>
{
fn get(&self, key: &H::Out) -> Option<DBValue> { hash_db::PlainDB::get(self, key) }
fn contains(&self, key: &H::Out) -> bool { hash_db::PlainDB::contains(self, key) }
}
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: Hasher
> hash_db::HashDB<H, DBValue>
for Ephemeral<'a, S, H>
{
fn get(&self, key: &H::Out, prefix: Prefix) -> Option<DBValue> {
if let Some(val) = hash_db::HashDB::get(self.overlay, key, prefix) {
Some(val)
} else {
match self.storage.get(&key, prefix) {
Ok(x) => x,
Err(e) => {
warn!(target: "trie", "Failed to read from DB: {}", e);
None
},
}
}
}
fn contains(&self, key: &H::Out, prefix: Prefix) -> bool {
hash_db::HashDB::get(self, key, prefix).is_some()
}
fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out {
hash_db::HashDB::insert(self.overlay, prefix, value)
}
fn emplace(&mut self, key: H::Out, prefix: Prefix, value: DBValue) {
hash_db::HashDB::emplace(self.overlay, key, prefix, value)
}
fn remove(&mut self, key: &H::Out, prefix: Prefix) {
hash_db::HashDB::remove(self.overlay, key, prefix)
}
}
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: Hasher
> hash_db::HashDBRef<H, DBValue>
for Ephemeral<'a, S, H>
{
fn get(&self, key: &H::Out, prefix: Prefix) -> Option<DBValue> { hash_db::HashDB::get(self, key, prefix) }
fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { hash_db::HashDB::contains(self, key, prefix) }
}
/// Key-value pairs storage that is used by trie backend essence.
pub trait TrieBackendStorage<H: Hasher>: Send + Sync {
/// Type of in-memory overlay.
type Overlay: hash_db::HashDB<H, DBValue> + Default + Consolidate;
/// Get the value stored at key.
fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String>;
}
// This implementation is used by normal storage trie clients.
impl<H: Hasher> TrieBackendStorage<H> for Arc<dyn Storage<H>> {
type Overlay = PrefixedMemoryDB<H>;
fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
Storage::<H>::get(self.deref(), key, prefix)
}
}
// This implementation is used by test storage trie clients.
impl<H: Hasher> TrieBackendStorage<H> for PrefixedMemoryDB<H> {
type Overlay = PrefixedMemoryDB<H>;
fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
Ok(hash_db::HashDB::get(self, key, prefix))
}
}
impl<H: Hasher> TrieBackendStorage<H> for MemoryDB<H> {
type Overlay = MemoryDB<H>;
fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
Ok(hash_db::HashDB::get(self, key, prefix))
}
}