Remove requirement on Hash = H256, make Proposer return StorageChanges and Proof (#3860)

* Extend `Proposer` to optionally generate a proof of the proposal

* Something

* Refactor sr-api to not depend on client anymore

* Fix benches

* Apply suggestions from code review

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Apply suggestions from code review

* Introduce new `into_storage_changes` function

* Switch to runtime api for `execute_block` and don't require `H256`
anywhere in the code

* Put the `StorageChanges` into the `Proposal`

* Move the runtime api error to its own trait

* Adds `StorageTransactionCache` to the runtime api

This requires that we add `type NodeBlock = ` to the
`impl_runtime_apis!` macro to work around some bugs in rustc :(

* Remove `type NodeBlock` and switch to a "better" hack

* Start using the transaction cache from the runtime api

* Make it compile

* Move `InMemory` to its own file

* Make all tests work again

* Return block, storage_changes and proof from Blockbuilder::bake()

* Make sure that we use/set `storage_changes` when possible

* Add test

* Fix deadlock

* Remove accidentally added folders

* Introduce `RecordProof` as argument type to be more explicit

* Update client/src/client.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update primitives/state-machine/src/ext.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Integrates review feedback

* Remove `unsafe` usage

* Update client/block-builder/src/lib.rs

Co-Authored-By: Benjamin Kampmann <ben@gnunicorn.org>

* Update client/src/call_executor.rs

* Bump versions

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>
Co-authored-by: Benjamin Kampmann <ben.kampmann@googlemail.com>
This commit is contained in:
Bastian Köcher
2020-01-10 10:48:32 +01:00
committed by GitHub
parent 74d6e660c6
commit fd6b29dd2c
140 changed files with 4860 additions and 3339 deletions
@@ -16,17 +16,13 @@
//! State machine backends. These manage the code and storage of contracts.
use std::{error, fmt, cmp::Ord, collections::{HashMap, BTreeMap}, marker::PhantomData, ops};
use log::warn;
use hash_db::Hasher;
use crate::trie_backend::TrieBackend;
use crate::trie_backend_essence::TrieBackendStorage;
use sp_trie::{
TrieMut, MemoryDB, child_trie_root, default_child_trie_root, TrieConfiguration,
trie_types::{TrieDBMut, Layout},
};
use codec::{Encode, Codec};
use sp_core::storage::{ChildInfo, OwnedChildInfo, Storage};
use sp_trie::{TrieMut, MemoryDB, trie_types::TrieDBMut};
use codec::Encode;
use sp_core::storage::{ChildInfo, OwnedChildInfo};
/// A state backend is used to read state data and can have changes committed
/// to it.
@@ -37,7 +33,7 @@ pub trait Backend<H: Hasher>: std::fmt::Debug {
type Error: super::Error;
/// Storage changes to be applied if committing
type Transaction: Consolidate + Default;
type Transaction: Consolidate + Default + Send;
/// Type of trie backend storage.
type TrieBackendStorage: TrieBackendStorage<H>;
@@ -317,329 +313,6 @@ impl<H: Hasher, KF: sp_trie::KeyFunction<H>> Consolidate for sp_trie::GenericMem
}
}
/// Error impossible.
// FIXME: use `!` type when stabilized. https://github.com/rust-lang/rust/issues/35121
#[derive(Debug)]
pub enum Void {}
impl fmt::Display for Void {
fn fmt(&self, _: &mut fmt::Formatter) -> fmt::Result {
match *self {}
}
}
impl error::Error for Void {
fn description(&self) -> &str { "unreachable error" }
}
/// In-memory backend. Fully recomputes tries each time `as_trie_backend` is called but useful for
/// tests and proof checking.
pub struct InMemory<H: Hasher> {
inner: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>,
// This field is only needed for returning reference in `as_trie_backend`.
trie: Option<TrieBackend<MemoryDB<H>, H>>,
_hasher: PhantomData<H>,
}
impl<H: Hasher> std::fmt::Debug for InMemory<H> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "InMemory ({} values)", self.inner.len())
}
}
impl<H: Hasher> Default for InMemory<H> {
fn default() -> Self {
InMemory {
inner: Default::default(),
trie: None,
_hasher: PhantomData,
}
}
}
impl<H: Hasher> Clone for InMemory<H> {
fn clone(&self) -> Self {
InMemory {
inner: self.inner.clone(),
trie: None,
_hasher: PhantomData,
}
}
}
impl<H: Hasher> PartialEq for InMemory<H> {
fn eq(&self, other: &Self) -> bool {
self.inner.eq(&other.inner)
}
}
impl<H: Hasher> InMemory<H> where H::Out: Codec {
/// Copy the state, with applied updates
pub fn update(&self, changes: <Self as Backend<H>>::Transaction) -> Self {
let mut inner = self.inner.clone();
for (child_info, key_values) in changes {
let entry = inner.entry(child_info).or_default();
for (key, val) in key_values {
match val {
Some(v) => { entry.insert(key, v); },
None => { entry.remove(&key); },
}
}
}
inner.into()
}
}
impl<H: Hasher> From<HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>> for InMemory<H> {
fn from(inner: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>) -> Self {
InMemory {
inner: inner,
trie: None,
_hasher: PhantomData,
}
}
}
impl<H: Hasher> From<Storage> for InMemory<H> {
fn from(inners: Storage) -> Self {
let mut inner: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>
= inners.children.into_iter().map(|(k, c)| (Some((k, c.child_info)), c.data)).collect();
inner.insert(None, inners.top);
InMemory {
inner: inner,
trie: None,
_hasher: PhantomData,
}
}
}
impl<H: Hasher> From<BTreeMap<Vec<u8>, Vec<u8>>> for InMemory<H> {
fn from(inner: BTreeMap<Vec<u8>, Vec<u8>>) -> Self {
let mut expanded = HashMap::new();
expanded.insert(None, inner);
InMemory {
inner: expanded,
trie: None,
_hasher: PhantomData,
}
}
}
impl<H: Hasher> From<Vec<(Option<(Vec<u8>, OwnedChildInfo)>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>>
for InMemory<H> {
fn from(
inner: Vec<(Option<(Vec<u8>, OwnedChildInfo)>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>,
) -> Self {
let mut expanded: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>
= HashMap::new();
for (child_info, key_values) in inner {
let entry = expanded.entry(child_info).or_default();
for (key, value) in key_values {
if let Some(value) = value {
entry.insert(key, value);
}
}
}
expanded.into()
}
}
impl<H: Hasher> InMemory<H> {
/// child storage key iterator
pub fn child_storage_keys(&self) -> impl Iterator<Item=(&[u8], ChildInfo)> {
self.inner.iter().filter_map(|item|
item.0.as_ref().map(|v|(&v.0[..], v.1.as_ref()))
)
}
}
impl<H: Hasher> Backend<H> for InMemory<H> where H::Out: Codec {
type Error = Void;
type Transaction = Vec<(
Option<(Vec<u8>, OwnedChildInfo)>,
Vec<(Vec<u8>, Option<Vec<u8>>)>,
)>;
type TrieBackendStorage = MemoryDB<H>;
fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
Ok(self.inner.get(&None).and_then(|map| map.get(key).map(Clone::clone)))
}
fn child_storage(
&self,
storage_key: &[u8],
child_info: ChildInfo,
key: &[u8],
) -> Result<Option<Vec<u8>>, Self::Error> {
Ok(self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
.and_then(|map| map.get(key).map(Clone::clone)))
}
fn exists_storage(&self, key: &[u8]) -> Result<bool, Self::Error> {
Ok(self.inner.get(&None).map(|map| map.get(key).is_some()).unwrap_or(false))
}
fn next_storage_key(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
let range = (ops::Bound::Excluded(key), ops::Bound::Unbounded);
let next_key = self.inner.get(&None)
.and_then(|map| map.range::<[u8], _>(range).next().map(|(k, _)| k).cloned());
Ok(next_key)
}
fn next_child_storage_key(
&self,
storage_key: &[u8],
child_info: ChildInfo,
key: &[u8],
) -> Result<Option<Vec<u8>>, Self::Error> {
let range = (ops::Bound::Excluded(key), ops::Bound::Unbounded);
let next_key = self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
.and_then(|map| map.range::<[u8], _>(range).next().map(|(k, _)| k).cloned());
Ok(next_key)
}
fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
self.inner.get(&None).map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f));
}
fn for_key_values_with_prefix<F: FnMut(&[u8], &[u8])>(&self, prefix: &[u8], mut f: F) {
self.inner.get(&None).map(|map| map.iter().filter(|(key, _val)| key.starts_with(prefix))
.for_each(|(k, v)| f(k, v)));
}
fn for_keys_in_child_storage<F: FnMut(&[u8])>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
mut f: F,
) {
self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
.map(|map| map.keys().for_each(|k| f(&k)));
}
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
prefix: &[u8],
f: F,
) {
self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
.map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f));
}
fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
<H as Hasher>::Out: Ord,
{
let existing_pairs = self.inner.get(&None)
.into_iter()
.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
let transaction: Vec<_> = delta.into_iter().collect();
let root = Layout::<H>::trie_root(existing_pairs.chain(transaction.iter().cloned())
.collect::<HashMap<_, _>>()
.into_iter()
.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
);
let full_transaction = transaction.into_iter().collect();
(root, vec![(None, full_transaction)])
}
fn child_storage_root<I>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
delta: I,
) -> (H::Out, bool, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
H::Out: Ord
{
let storage_key = storage_key.to_vec();
let child_info = Some((storage_key.clone(), child_info.to_owned()));
let existing_pairs = self.inner.get(&child_info)
.into_iter()
.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
let transaction: Vec<_> = delta.into_iter().collect();
let root = child_trie_root::<Layout<H>, _, _, _>(
&storage_key,
existing_pairs.chain(transaction.iter().cloned())
.collect::<HashMap<_, _>>()
.into_iter()
.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
);
let full_transaction = transaction.into_iter().collect();
let is_default = root == default_child_trie_root::<Layout<H>>(&storage_key);
(root, is_default, vec![(child_info, full_transaction)])
}
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
self.inner.get(&None)
.into_iter()
.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone())))
.collect()
}
fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
self.inner.get(&None)
.into_iter()
.flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned())
.collect()
}
fn child_keys(
&self,
storage_key: &[u8],
child_info: ChildInfo,
prefix: &[u8],
) -> Vec<Vec<u8>> {
self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
.into_iter()
.flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned())
.collect()
}
fn as_trie_backend(&mut self)-> Option<&TrieBackend<Self::TrieBackendStorage, H>> {
let mut mdb = MemoryDB::default();
let mut new_child_roots = Vec::new();
let mut root_map = None;
for (child_info, map) in &self.inner {
if let Some((storage_key, _child_info)) = child_info.as_ref() {
// no need to use child_info at this point because we use a MemoryDB for
// proof (with PrefixedMemoryDB it would be needed).
let ch = insert_into_memory_db::<H, _>(&mut mdb, map.clone().into_iter())?;
new_child_roots.push((storage_key.clone(), ch.as_ref().into()));
} else {
root_map = Some(map);
}
}
let root = match root_map {
Some(map) => insert_into_memory_db::<H, _>(
&mut mdb,
map.clone().into_iter().chain(new_child_roots.into_iter()),
)?,
None => insert_into_memory_db::<H, _>(
&mut mdb,
new_child_roots.into_iter(),
)?,
};
self.trie = Some(TrieBackend::new(mdb, root));
self.trie.as_ref()
}
}
/// Insert input pairs into memory db.
pub(crate) fn insert_into_memory_db<H, I>(mdb: &mut MemoryDB<H>, input: I) -> Option<H::Out>
where
@@ -659,25 +332,3 @@ pub(crate) fn insert_into_memory_db<H, I>(mdb: &mut MemoryDB<H>, input: I) -> Op
Some(root)
}
#[cfg(test)]
mod tests {
use super::*;
/// Assert in memory backend with only child trie keys works as trie backend.
#[test]
fn in_memory_with_child_trie_only() {
let storage = InMemory::<sp_core::Blake2Hasher>::default();
let child_info = OwnedChildInfo::new_default(b"unique_id_1".to_vec());
let mut storage = storage.update(
vec![(
Some((b"1".to_vec(), child_info.clone())),
vec![(b"2".to_vec(), Some(b"3".to_vec()))]
)]
);
let trie_backend = storage.as_trie_backend().unwrap();
assert_eq!(trie_backend.child_storage(b"1", child_info.as_ref(), b"2").unwrap(),
Some(b"3".to_vec()));
assert!(trie_backend.storage(b"1").unwrap().is_some());
}
}
@@ -19,7 +19,7 @@
use std::{
collections::BTreeMap, any::{TypeId, Any}, iter::FromIterator, ops::Bound
};
use crate::backend::{Backend, InMemory};
use crate::{Backend, InMemoryBackend};
use hash_db::Hasher;
use sp_trie::{TrieConfiguration, default_child_trie_root};
use sp_trie::trie_types::Layout;
@@ -288,7 +288,7 @@ impl Externalities for BasicExternalities {
if let Some(child) = self.inner.children.get(storage_key.as_ref()) {
let delta = child.data.clone().into_iter().map(|(k, v)| (k, Some(v)));
InMemory::<Blake2Hasher>::default()
InMemoryBackend::<Blake2Hasher>::default()
.child_storage_root(storage_key.as_ref(), child.child_info.as_ref(), delta).0
} else {
default_child_trie_root::<Layout<Blake2Hasher>>(storage_key.as_ref())
@@ -339,9 +339,9 @@ fn prepare_digest_input<'a, H, Number>(
mod test {
use codec::Encode;
use sp_core::Blake2Hasher;
use sp_core::storage::well_known_keys::{EXTRINSIC_INDEX};
use sp_core::storage::well_known_keys::EXTRINSIC_INDEX;
use sp_core::storage::ChildInfo;
use crate::backend::InMemory;
use crate::InMemoryBackend;
use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage};
use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData};
use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet};
@@ -351,20 +351,20 @@ mod test {
const CHILD_INFO_2: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_2");
fn prepare_for_build(zero: u64) -> (
InMemory<Blake2Hasher>,
InMemoryBackend<Blake2Hasher>,
InMemoryStorage<Blake2Hasher, u64>,
OverlayedChanges,
Configuration,
) {
let config = Configuration { digest_interval: 4, digest_levels: 2 };
let backend: InMemory<_> = vec![
let backend: InMemoryBackend<_> = vec![
(vec![100], vec![255]),
(vec![101], vec![255]),
(vec![102], vec![255]),
(vec![103], vec![255]),
(vec![104], vec![255]),
(vec![105], vec![255]),
].into_iter().collect::<::std::collections::BTreeMap<_, _>>().into();
].into_iter().collect::<std::collections::BTreeMap<_, _>>().into();
let child_trie_key1 = b"1".to_vec();
let child_trie_key2 = b"2".to_vec();
let storage = InMemoryStorage::with_inputs(vec![
@@ -84,37 +84,37 @@ pub const NO_EXTRINSIC_INDEX: u32 = 0xffffffff;
/// Requirements for block number that can be used with changes tries.
pub trait BlockNumber:
Send + Sync + 'static +
::std::fmt::Display +
std::fmt::Display +
Clone +
From<u32> + TryInto<u32> + One + Zero +
PartialEq + Ord +
::std::hash::Hash +
::std::ops::Add<Self, Output=Self> + ::std::ops::Sub<Self, Output=Self> +
::std::ops::Mul<Self, Output=Self> + ::std::ops::Div<Self, Output=Self> +
::std::ops::Rem<Self, Output=Self> +
::std::ops::AddAssign<Self> +
std::hash::Hash +
std::ops::Add<Self, Output=Self> + ::std::ops::Sub<Self, Output=Self> +
std::ops::Mul<Self, Output=Self> + ::std::ops::Div<Self, Output=Self> +
std::ops::Rem<Self, Output=Self> +
std::ops::AddAssign<Self> +
num_traits::CheckedMul + num_traits::CheckedSub +
Decode + Encode
{}
impl<T> BlockNumber for T where T:
Send + Sync + 'static +
::std::fmt::Display +
std::fmt::Display +
Clone +
From<u32> + TryInto<u32> + One + Zero +
PartialEq + Ord +
::std::hash::Hash +
::std::ops::Add<Self, Output=Self> + ::std::ops::Sub<Self, Output=Self> +
::std::ops::Mul<Self, Output=Self> + ::std::ops::Div<Self, Output=Self> +
::std::ops::Rem<Self, Output=Self> +
::std::ops::AddAssign<Self> +
std::hash::Hash +
std::ops::Add<Self, Output=Self> + ::std::ops::Sub<Self, Output=Self> +
std::ops::Mul<Self, Output=Self> + ::std::ops::Div<Self, Output=Self> +
std::ops::Rem<Self, Output=Self> +
std::ops::AddAssign<Self> +
num_traits::CheckedMul + num_traits::CheckedSub +
Decode + Encode,
{}
/// Block identifier that could be used to determine fork of this block.
#[derive(Debug)]
pub struct AnchorBlockId<Hash: ::std::fmt::Debug, Number: BlockNumber> {
pub struct AnchorBlockId<Hash: std::fmt::Debug, Number: BlockNumber> {
/// Hash of this block.
pub hash: Hash,
/// Number of this block.
@@ -173,12 +173,14 @@ pub struct ConfigurationRange<'a, N> {
/// Compute the changes trie root and transaction for given block.
/// Returns Err(()) if unknown `parent_hash` has been passed.
/// Returns Ok(None) if there's no data to perform computation.
/// Panics if background storage returns an error OR if insert to MemoryDB fails.
/// Panics if background storage returns an error (and `panic_on_storage_error` is `true`) OR
/// if insert to MemoryDB fails.
pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, Number: BlockNumber>(
backend: &B,
storage: Option<&'a S>,
changes: &OverlayedChanges,
parent_hash: H::Out,
panic_on_storage_error: bool,
) -> Result<Option<(MemoryDB<H>, H::Out, CacheAction<H::Out, Number>)>, ()>
where
H::Out: Ord + 'static + Encode,
@@ -188,6 +190,19 @@ pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, N
_ => return Ok(None),
};
/// Panics when `res.is_err() && panic`, otherwise it returns `Err(())` on an error.
fn maybe_panic<R, E: std::fmt::Debug>(
res: std::result::Result<R, E>,
panic: bool,
) -> std::result::Result<R, ()> {
res.map(Ok)
.unwrap_or_else(|e| if panic {
panic!("changes trie: storage access is not allowed to fail within runtime: {:?}", e)
} else {
Err(())
})
}
// FIXME: remove this in https://github.com/paritytech/substrate/pull/3201
let config = ConfigurationRange {
config,
@@ -200,13 +215,16 @@ pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, N
let block = parent.number.clone() + One::one();
// storage errors are considered fatal (similar to situations when runtime fetches values from storage)
let (input_pairs, child_input_pairs, digest_input_blocks) = prepare_input::<B, H, Number>(
backend,
storage,
config.clone(),
changes,
&parent,
).expect("changes trie: storage access is not allowed to fail within runtime");
let (input_pairs, child_input_pairs, digest_input_blocks) = maybe_panic(
prepare_input::<B, H, Number>(
backend,
storage,
config.clone(),
changes,
&parent,
),
panic_on_storage_error,
)?;
// prepare cached data
let mut cache_action = prepare_cached_build_data(config, block.clone());
@@ -230,8 +248,7 @@ pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, N
let (key, value) = input_pair.into();
not_empty = true;
trie.insert(&key, &value)
.expect("changes trie: insertion to trie is not allowed to fail within runtime");
maybe_panic(trie.insert(&key, &value), panic_on_storage_error)?;
}
cache_action = cache_action.insert(
@@ -247,8 +264,7 @@ pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, N
{
let mut trie = TrieDBMut::<H>::new(&mut mdb, &mut root);
for (key, value) in child_roots.into_iter().map(Into::into) {
trie.insert(&key, &value)
.expect("changes trie: insertion to trie is not allowed to fail within runtime");
maybe_panic(trie.insert(&key, &value), panic_on_storage_error)?;
}
let mut storage_changed_keys = HashSet::new();
@@ -260,9 +276,9 @@ pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, N
}
let (key, value) = input_pair.into();
trie.insert(&key, &value)
.expect("changes trie: insertion to trie is not allowed to fail within runtime");
maybe_panic(trie.insert(&key, &value), panic_on_storage_error)?;
}
cache_action = cache_action.insert(
None,
storage_changed_keys,
@@ -38,7 +38,7 @@ pub struct InMemoryStorage<H: Hasher, Number: BlockNumber> {
/// Adapter for using changes trie storage as a TrieBackendEssence' storage.
pub struct TrieBackendAdapter<'a, H: Hasher, Number: BlockNumber> {
storage: &'a dyn Storage<H, Number>,
_hasher: ::std::marker::PhantomData<(H, Number)>,
_hasher: std::marker::PhantomData<(H, Number)>,
}
struct InMemoryStorageData<H: Hasher, Number: BlockNumber> {
+53 -100
View File
@@ -17,18 +17,16 @@
//! Concrete externalities implementation.
use crate::{
backend::Backend, OverlayedChanges,
changes_trie::{
Storage as ChangesTrieStorage, CacheAction as ChangesTrieCacheAction, build_changes_trie,
},
backend::Backend, OverlayedChanges, StorageTransactionCache,
changes_trie::Storage as ChangesTrieStorage,
};
use hash_db::Hasher;
use sp_core::{
storage::{ChildStorageKey, well_known_keys::is_child_storage_key, ChildInfo},
traits::Externalities, hexdisplay::HexDisplay, hash::H256,
traits::Externalities, hexdisplay::HexDisplay,
};
use sp_trie::{trie_types::Layout, MemoryDB, default_child_trie_root};
use sp_trie::{trie_types::Layout, default_child_trie_root};
use sp_externalities::Extensions;
use codec::{Decode, Encode};
@@ -67,23 +65,20 @@ impl<B: error::Error, E: error::Error> error::Error for Error<B, E> {
}
/// Wraps a read-only backend, call executor, and current overlayed changes.
pub struct Ext<'a, H, N, B, T> where H: Hasher<Out=H256>, B: 'a + Backend<H> {
pub struct Ext<'a, H, N, B, T>
where
H: Hasher,
B: 'a + Backend<H>,
N: crate::changes_trie::BlockNumber,
{
/// The overlayed changes to write to.
overlay: &'a mut OverlayedChanges,
/// The storage backend to read from.
backend: &'a B,
/// The storage transaction necessary to commit to the backend. Is cached when
/// `storage_root` is called and the cache is cleared on every subsequent change.
storage_transaction: Option<(B::Transaction, H::Out)>,
/// The cache for the storage transactions.
storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H, N>,
/// Changes trie storage to read from.
changes_trie_storage: Option<&'a T>,
/// The changes trie transaction necessary to commit to the changes trie backend.
/// Set to Some when `storage_changes_root` is called. Could be replaced later
/// by calling `storage_changes_root` again => never used as cache.
/// This differs from `storage_transaction` behavior, because the moment when
/// `storage_changes_root` is called matters + we need to remember additional
/// data at this moment (block number).
changes_trie_transaction: Option<(MemoryDB<H>, H::Out, ChangesTrieCacheAction<H::Out, N>)>,
/// Pseudo-unique id used for tracing.
pub id: u16,
/// Dummy usage of N arg.
@@ -94,7 +89,8 @@ pub struct Ext<'a, H, N, B, T> where H: Hasher<Out=H256>, B: 'a + Backend<H> {
impl<'a, H, N, B, T> Ext<'a, H, N, B, T>
where
H: Hasher<Out=H256>,
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
B: 'a + Backend<H>,
T: 'a + ChangesTrieStorage<H, N>,
N: crate::changes_trie::BlockNumber,
@@ -103,6 +99,7 @@ where
/// Create a new `Ext` from overlayed changes and read-only backend
pub fn new(
overlay: &'a mut OverlayedChanges,
storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H, N>,
backend: &'a B,
changes_trie_storage: Option<&'a T>,
extensions: Option<&'a mut Extensions>,
@@ -110,49 +107,27 @@ where
Ext {
overlay,
backend,
storage_transaction: None,
changes_trie_storage,
changes_trie_transaction: None,
storage_transaction_cache,
id: rand::random(),
_phantom: Default::default(),
extensions,
}
}
/// Get the transaction necessary to update the backend.
pub fn transaction(&mut self) -> (
(B::Transaction, H256),
Option<crate::ChangesTrieTransaction<H, N>>,
) {
let _ = self.storage_root();
let (storage_transaction, changes_trie_transaction) = (
self.storage_transaction
.take()
.expect("storage_transaction always set after calling storage root; qed"),
self.changes_trie_transaction
.take()
.map(|(tx, _, cache)| (tx, cache)),
);
(
storage_transaction,
changes_trie_transaction,
)
}
/// Invalidates the currently cached storage root and the db transaction.
///
/// Called when there are changes that likely will invalidate the storage root.
fn mark_dirty(&mut self) {
self.storage_transaction = None;
self.storage_transaction_cache.reset();
}
}
#[cfg(test)]
impl<'a, H, N, B, T> Ext<'a, H, N, B, T>
where
H: Hasher<Out=H256>,
H: Hasher,
H::Out: Ord + 'static,
B: 'a + Backend<H>,
T: 'a + ChangesTrieStorage<H, N>,
N: crate::changes_trie::BlockNumber,
@@ -173,7 +148,8 @@ where
impl<'a, H, B, T, N> Externalities for Ext<'a, H, N, B, T>
where
H: Hasher<Out=H256>,
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
B: 'a + Backend<H>,
T: 'a + ChangesTrieStorage<H, N>,
N: crate::changes_trie::BlockNumber,
@@ -295,6 +271,7 @@ where
HexDisplay::from(&key),
result.as_ref().map(HexDisplay::from),
);
result
}
@@ -330,8 +307,8 @@ where
HexDisplay::from(&key),
result,
);
result
result
}
fn exists_child_storage(
@@ -499,7 +476,7 @@ where
fn storage_root(&mut self) -> Vec<u8> {
let _guard = sp_panic_handler::AbortGuard::force_abort();
if let Some((_, ref root)) = self.storage_transaction {
if let Some(ref root) = self.storage_transaction_cache.transaction_storage_root {
trace!(target: "state-trace", "{:04x}: Root (cached) {}",
self.id,
HexDisplay::from(&root.as_ref()),
@@ -507,35 +484,8 @@ where
return root.encode();
}
let child_storage_keys = self.overlay.prospective.children.keys()
.chain(self.overlay.committed.children.keys());
let child_delta_iter = child_storage_keys.map(|storage_key|
(
storage_key.clone(),
self.overlay.committed.children.get(storage_key)
.into_iter()
.flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))
.chain(
self.overlay.prospective.children.get(storage_key)
.into_iter()
.flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))
),
self.overlay.child_info(storage_key).cloned()
.expect("child info initialized in either committed or prospective"),
)
);
// compute and memoize
let delta = self.overlay.committed.top.iter().map(|(k, v)| (k.clone(), v.value.clone()))
.chain(self.overlay.prospective.top.iter().map(|(k, v)| (k.clone(), v.value.clone())));
let (root, transaction) = self.backend.full_storage_root(delta, child_delta_iter);
self.storage_transaction = Some((transaction, root));
trace!(target: "state-trace", "{:04x}: Root {}",
self.id,
HexDisplay::from(&root.as_ref()),
);
let root = self.overlay.storage_root(self.backend, self.storage_transaction_cache);
trace!(target: "state-trace", "{:04x}: Root {}", self.id, HexDisplay::from(&root.as_ref()));
root.encode()
}
@@ -544,7 +494,7 @@ where
storage_key: ChildStorageKey,
) -> Vec<u8> {
let _guard = sp_panic_handler::AbortGuard::force_abort();
if self.storage_transaction.is_some() {
if self.storage_transaction_cache.transaction_storage_root.is_some() {
let root = self
.storage(storage_key.as_ref())
.and_then(|k| Decode::decode(&mut &k[..]).ok())
@@ -612,35 +562,33 @@ where
fn storage_changes_root(&mut self, parent_hash: &[u8]) -> Result<Option<Vec<u8>>, ()> {
let _guard = sp_panic_handler::AbortGuard::force_abort();
self.changes_trie_transaction = build_changes_trie::<_, T, H, N>(
let root = self.overlay.changes_trie_root(
self.backend,
self.changes_trie_storage.clone(),
self.overlay,
H256::decode(&mut &parent_hash[..]).map_err(|e|
Decode::decode(&mut &parent_hash[..]).map_err(|e|
trace!(
target: "state-trace",
"Failed to decode changes root parent hash: {}",
e,
)
)?,
)?;
let result = Ok(
self.changes_trie_transaction.as_ref().map(|(_, root, _)| root.encode())
true,
self.storage_transaction_cache,
);
trace!(target: "state-trace", "{:04x}: ChangesRoot({}) {:?}",
self.id,
HexDisplay::from(&parent_hash.as_ref()),
result,
HexDisplay::from(&parent_hash),
root,
);
result
root.map(|r| r.map(|o| o.encode()))
}
}
impl<'a, H, B, T, N> sp_externalities::ExtensionStore for Ext<'a, H, N, B, T>
where
H: Hasher<Out=H256>,
H: Hasher,
B: 'a + Backend<H>,
T: 'a + ChangesTrieStorage<H, N>,
N: crate::changes_trie::BlockNumber,
@@ -655,16 +603,16 @@ mod tests {
use super::*;
use hex_literal::hex;
use codec::Encode;
use sp_core::{Blake2Hasher, storage::well_known_keys::EXTRINSIC_INDEX, map};
use sp_core::{H256, Blake2Hasher, storage::well_known_keys::EXTRINSIC_INDEX, map};
use crate::{
changes_trie::{
Configuration as ChangesTrieConfiguration,
InMemoryStorage as InMemoryChangesTrieStorage,
}, backend::InMemory, overlayed_changes::OverlayedValue,
}, InMemoryBackend, overlayed_changes::OverlayedValue,
};
use sp_core::storage::{Storage, StorageChild};
type TestBackend = InMemory<Blake2Hasher>;
type TestBackend = InMemoryBackend<Blake2Hasher>;
type TestChangesTrieStorage = InMemoryChangesTrieStorage<Blake2Hasher, u64>;
type TestExt<'a> = Ext<'a, Blake2Hasher, u64, TestBackend, TestChangesTrieStorage>;
@@ -691,27 +639,30 @@ mod tests {
#[test]
fn storage_changes_root_is_none_when_storage_is_not_provided() {
let mut overlay = prepare_overlay_with_changes();
let mut cache = StorageTransactionCache::default();
let backend = TestBackend::default();
let mut ext = TestExt::new(&mut overlay, &backend, None, None);
let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None);
}
#[test]
fn storage_changes_root_is_none_when_extrinsic_changes_are_none() {
let mut overlay = prepare_overlay_with_changes();
let mut cache = StorageTransactionCache::default();
overlay.changes_trie_config = None;
let storage = TestChangesTrieStorage::with_blocks(vec![(100, Default::default())]);
let backend = TestBackend::default();
let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None);
let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, Some(&storage), None);
assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None);
}
#[test]
fn storage_changes_root_is_some_when_extrinsic_changes_are_non_empty() {
let mut overlay = prepare_overlay_with_changes();
let mut cache = StorageTransactionCache::default();
let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
let backend = TestBackend::default();
let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None);
let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, Some(&storage), None);
assert_eq!(
ext.storage_changes_root(&H256::default().encode()).unwrap(),
Some(hex!("bb0c2ef6e1d36d5490f9766cfcc7dfe2a6ca804504c3bb206053890d6dd02376").to_vec()),
@@ -721,10 +672,11 @@ mod tests {
#[test]
fn storage_changes_root_is_some_when_extrinsic_changes_are_empty() {
let mut overlay = prepare_overlay_with_changes();
let mut cache = StorageTransactionCache::default();
overlay.prospective.top.get_mut(&vec![1]).unwrap().value = None;
let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
let backend = TestBackend::default();
let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None);
let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, Some(&storage), None);
assert_eq!(
ext.storage_changes_root(&H256::default().encode()).unwrap(),
Some(hex!("96f5aae4690e7302737b6f9b7f8567d5bbb9eac1c315f80101235a92d9ec27f4").to_vec()),
@@ -733,6 +685,7 @@ mod tests {
#[test]
fn next_storage_key_works() {
let mut cache = StorageTransactionCache::default();
let mut overlay = OverlayedChanges::default();
overlay.set_storage(vec![20], None);
overlay.set_storage(vec![30], Some(vec![31]));
@@ -745,7 +698,7 @@ mod tests {
children: map![]
}.into();
let ext = TestExt::new(&mut overlay, &backend, None, None);
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
// next_backend < next_overlay
assert_eq!(ext.next_storage_key(&[5]), Some(vec![10]));
@@ -761,7 +714,7 @@ mod tests {
drop(ext);
overlay.set_storage(vec![50], Some(vec![50]));
let ext = TestExt::new(&mut overlay, &backend, None, None);
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
// next_overlay exist but next_backend doesn't exist
assert_eq!(ext.next_storage_key(&[40]), Some(vec![50]));
@@ -774,7 +727,7 @@ mod tests {
const CHILD_UUID_1: &[u8] = b"unique_id_1";
const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(CHILD_UUID_1);
let mut cache = StorageTransactionCache::default();
let child = || ChildStorageKey::from_slice(CHILD_KEY_1).unwrap();
let mut overlay = OverlayedChanges::default();
overlay.set_child_storage(child().as_ref().to_vec(), CHILD_INFO_1, vec![20], None);
@@ -794,7 +747,7 @@ mod tests {
}.into();
let ext = TestExt::new(&mut overlay, &backend, None, None);
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
// next_backend < next_overlay
assert_eq!(ext.next_child_storage_key(child(), CHILD_INFO_1, &[5]), Some(vec![10]));
@@ -810,7 +763,7 @@ mod tests {
drop(ext);
overlay.set_child_storage(child().as_ref().to_vec(), CHILD_INFO_1, vec![50], Some(vec![50]));
let ext = TestExt::new(&mut overlay, &backend, None, None);
let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
// next_overlay exist but next_backend doesn't exist
assert_eq!(ext.next_child_storage_key(child(), CHILD_INFO_1, &[40]), Some(vec![50]));
@@ -0,0 +1,378 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! State machine in memory backend.
use crate::{trie_backend::TrieBackend, backend::{Backend, insert_into_memory_db}};
use std::{error, fmt, collections::{BTreeMap, HashMap}, marker::PhantomData, ops};
use hash_db::Hasher;
use sp_trie::{
MemoryDB, child_trie_root, default_child_trie_root, TrieConfiguration, trie_types::Layout,
};
use codec::Codec;
use sp_core::storage::{ChildInfo, OwnedChildInfo, Storage};
/// Error impossible.
// FIXME: use `!` type when stabilized. https://github.com/rust-lang/rust/issues/35121
#[derive(Debug)]
pub enum Void {}
impl fmt::Display for Void {
fn fmt(&self, _: &mut fmt::Formatter) -> fmt::Result {
match *self {}
}
}
impl error::Error for Void {
fn description(&self) -> &str { "unreachable error" }
}
/// In-memory backend. Fully recomputes tries each time `as_trie_backend` is called but useful for
/// tests and proof checking.
pub struct InMemory<H: Hasher> {
inner: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>,
// This field is only needed for returning reference in `as_trie_backend`.
trie: Option<TrieBackend<MemoryDB<H>, H>>,
_hasher: PhantomData<H>,
}
impl<H: Hasher> std::fmt::Debug for InMemory<H> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "InMemory ({} values)", self.inner.len())
}
}
impl<H: Hasher> Default for InMemory<H> {
fn default() -> Self {
InMemory {
inner: Default::default(),
trie: None,
_hasher: PhantomData,
}
}
}
impl<H: Hasher> Clone for InMemory<H> {
fn clone(&self) -> Self {
InMemory {
inner: self.inner.clone(),
trie: None,
_hasher: PhantomData,
}
}
}
impl<H: Hasher> PartialEq for InMemory<H> {
fn eq(&self, other: &Self) -> bool {
self.inner.eq(&other.inner)
}
}
impl<H: Hasher> InMemory<H> {
/// Copy the state, with applied updates
pub fn update<
T: IntoIterator<Item = (Option<(Vec<u8>, OwnedChildInfo)>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>
>(
&self,
changes: T,
) -> Self {
let mut inner = self.inner.clone();
for (child_info, key_values) in changes.into_iter() {
let entry = inner.entry(child_info).or_default();
for (key, val) in key_values {
match val {
Some(v) => { entry.insert(key, v); },
None => { entry.remove(&key); },
}
}
}
inner.into()
}
}
impl<H: Hasher> From<HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>>
for InMemory<H>
{
fn from(inner: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>) -> Self {
InMemory {
inner,
trie: None,
_hasher: PhantomData,
}
}
}
impl<H: Hasher> From<Storage> for InMemory<H> {
fn from(inners: Storage) -> Self {
let mut inner: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>
= inners.children.into_iter().map(|(k, c)| (Some((k, c.child_info)), c.data)).collect();
inner.insert(None, inners.top);
InMemory {
inner,
trie: None,
_hasher: PhantomData,
}
}
}
impl<H: Hasher> From<BTreeMap<Vec<u8>, Vec<u8>>> for InMemory<H> {
fn from(inner: BTreeMap<Vec<u8>, Vec<u8>>) -> Self {
let mut expanded = HashMap::new();
expanded.insert(None, inner);
InMemory {
inner: expanded,
trie: None,
_hasher: PhantomData,
}
}
}
impl<H: Hasher> From<Vec<(Option<(Vec<u8>, OwnedChildInfo)>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>>
for InMemory<H> {
fn from(
inner: Vec<(Option<(Vec<u8>, OwnedChildInfo)>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>,
) -> Self {
let mut expanded: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>
= HashMap::new();
for (child_info, key_values) in inner {
let entry = expanded.entry(child_info).or_default();
for (key, value) in key_values {
if let Some(value) = value {
entry.insert(key, value);
}
}
}
expanded.into()
}
}
impl<H: Hasher> InMemory<H> {
/// child storage key iterator
pub fn child_storage_keys(&self) -> impl Iterator<Item=(&[u8], ChildInfo)> {
self.inner.iter().filter_map(|item|
item.0.as_ref().map(|v|(&v.0[..], v.1.as_ref()))
)
}
}
impl<H: Hasher> Backend<H> for InMemory<H> where H::Out: Codec {
type Error = Void;
type Transaction = Vec<(
Option<(Vec<u8>, OwnedChildInfo)>,
Vec<(Vec<u8>, Option<Vec<u8>>)>,
)>;
type TrieBackendStorage = MemoryDB<H>;
fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
Ok(self.inner.get(&None).and_then(|map| map.get(key).map(Clone::clone)))
}
fn child_storage(
&self,
storage_key: &[u8],
child_info: ChildInfo,
key: &[u8],
) -> Result<Option<Vec<u8>>, Self::Error> {
Ok(self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
.and_then(|map| map.get(key).map(Clone::clone)))
}
fn exists_storage(&self, key: &[u8]) -> Result<bool, Self::Error> {
Ok(self.inner.get(&None).map(|map| map.get(key).is_some()).unwrap_or(false))
}
fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
self.inner.get(&None)
.map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f));
}
fn for_key_values_with_prefix<F: FnMut(&[u8], &[u8])>(&self, prefix: &[u8], mut f: F) {
self.inner.get(&None).map(|map| map.iter().filter(|(key, _val)| key.starts_with(prefix))
.for_each(|(k, v)| f(k, v)));
}
fn for_keys_in_child_storage<F: FnMut(&[u8])>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
mut f: F,
) {
self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
.map(|map| map.keys().for_each(|k| f(&k)));
}
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
prefix: &[u8],
f: F,
) {
self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
.map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f));
}
fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
<H as Hasher>::Out: Ord,
{
let existing_pairs = self.inner.get(&None)
.into_iter()
.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
let transaction: Vec<_> = delta.into_iter().collect();
let root = Layout::<H>::trie_root(existing_pairs.chain(transaction.iter().cloned())
.collect::<HashMap<_, _>>()
.into_iter()
.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
);
let full_transaction = transaction.into_iter().collect();
(root, vec![(None, full_transaction)])
}
fn child_storage_root<I>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
delta: I,
) -> (H::Out, bool, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
H::Out: Ord
{
let storage_key = storage_key.to_vec();
let child_info = Some((storage_key.clone(), child_info.to_owned()));
let existing_pairs = self.inner.get(&child_info)
.into_iter()
.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
let transaction: Vec<_> = delta.into_iter().collect();
let root = child_trie_root::<Layout<H>, _, _, _>(
&storage_key,
existing_pairs.chain(transaction.iter().cloned())
.collect::<HashMap<_, _>>()
.into_iter()
.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
);
let full_transaction = transaction.into_iter().collect();
let is_default = root == default_child_trie_root::<Layout<H>>(&storage_key);
(root, is_default, vec![(child_info, full_transaction)])
}
fn next_storage_key(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
let range = (ops::Bound::Excluded(key), ops::Bound::Unbounded);
let next_key = self.inner.get(&None)
.and_then(|map| map.range::<[u8], _>(range).next().map(|(k, _)| k).cloned());
Ok(next_key)
}
fn next_child_storage_key(
&self,
storage_key: &[u8],
child_info: ChildInfo,
key: &[u8],
) -> Result<Option<Vec<u8>>, Self::Error> {
let range = (ops::Bound::Excluded(key), ops::Bound::Unbounded);
let next_key = self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
.and_then(|map| map.range::<[u8], _>(range).next().map(|(k, _)| k).cloned());
Ok(next_key)
}
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
self.inner.get(&None)
.into_iter()
.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone())))
.collect()
}
fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
self.inner.get(&None)
.into_iter()
.flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned())
.collect()
}
fn child_keys(
&self,
storage_key: &[u8],
child_info: ChildInfo,
prefix: &[u8],
) -> Vec<Vec<u8>> {
self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
.into_iter()
.flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned())
.collect()
}
fn as_trie_backend(&mut self)-> Option<&TrieBackend<Self::TrieBackendStorage, H>> {
let mut mdb = MemoryDB::default();
let mut new_child_roots = Vec::new();
let mut root_map = None;
for (child_info, map) in &self.inner {
if let Some((storage_key, _child_info)) = child_info.as_ref() {
// no need to use child_info at this point because we use a MemoryDB for
// proof (with PrefixedMemoryDB it would be needed).
let ch = insert_into_memory_db::<H, _>(&mut mdb, map.clone().into_iter())?;
new_child_roots.push((storage_key.clone(), ch.as_ref().into()));
} else {
root_map = Some(map);
}
}
let root = match root_map {
Some(map) => insert_into_memory_db::<H, _>(
&mut mdb,
map.clone().into_iter().chain(new_child_roots.into_iter()),
)?,
None => insert_into_memory_db::<H, _>(
&mut mdb,
new_child_roots.into_iter(),
)?,
};
self.trie = Some(TrieBackend::new(mdb, root));
self.trie.as_ref()
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Assert in memory backend with only child trie keys works as trie backend.
#[test]
fn in_memory_with_child_trie_only() {
let storage = InMemory::<sp_core::Blake2Hasher>::default();
let child_info = OwnedChildInfo::new_default(b"unique_id_1".to_vec());
let mut storage = storage.update(
vec![(
Some((b"1".to_vec(), child_info.clone())),
vec![(b"2".to_vec(), Some(b"3".to_vec()))]
)]
);
let trie_backend = storage.as_trie_backend().unwrap();
assert_eq!(trie_backend.child_storage(b"1", child_info.as_ref(), b"2").unwrap(),
Some(b"3".to_vec()));
assert!(trie_backend.storage(b"1").unwrap().is_some());
}
}
+103 -106
View File
@@ -24,12 +24,13 @@ use hash_db::Hasher;
use codec::{Decode, Encode, Codec};
use sp_core::{
storage::{well_known_keys, ChildInfo}, NativeOrEncoded, NeverNativeValue,
traits::CodeExecutor, hexdisplay::HexDisplay, hash::H256,
traits::CodeExecutor, hexdisplay::HexDisplay
};
use overlayed_changes::OverlayedChangeSet;
use sp_externalities::Extensions;
pub mod backend;
mod in_memory_backend;
mod changes_trie;
mod error;
mod ext;
@@ -56,8 +57,9 @@ pub use changes_trie::{
key_changes, key_changes_proof, key_changes_proof_check,
prune as prune_changes_tries,
oldest_non_pruned_trie as oldest_non_pruned_changes_trie,
BlockNumber as ChangesTrieBlockNumber,
};
pub use overlayed_changes::OverlayedChanges;
pub use overlayed_changes::{OverlayedChanges, StorageChanges, StorageTransactionCache};
pub use proving_backend::{
create_proof_check_backend, create_proof_check_backend_storage, merge_storage_proofs,
ProofRecorder, ProvingBackend, ProvingBackendRecorder, StorageProof,
@@ -65,6 +67,7 @@ pub use proving_backend::{
pub use trie_backend_essence::{TrieBackendStorage, Storage};
pub use trie_backend::TrieBackend;
pub use error::{Error, ExecutionError};
pub use in_memory_backend::InMemory as InMemoryBackend;
type CallResult<R, E> = Result<NativeOrEncoded<R>, E>;
@@ -140,8 +143,10 @@ impl ExecutionStrategy {
warn!(
"Consensus error between wasm {:?} and native {:?}. Using wasm.",
wasm_result,
native_result
native_result,
);
warn!(" Native result {:?}", native_result);
warn!(" Wasm result {:?}", wasm_result);
wasm_result
}),
}
@@ -164,7 +169,12 @@ fn always_untrusted_wasm<E, R: Decode>() -> ExecutionManager<DefaultHandler<R, E
}
/// The substrate state machine.
pub struct StateMachine<'a, B, H, N, T, Exec> where H: Hasher<Out=H256>, B: Backend<H> {
pub struct StateMachine<'a, B, H, N, T, Exec>
where
H: Hasher,
B: Backend<H>,
N: ChangesTrieBlockNumber,
{
backend: &'a B,
exec: &'a Exec,
method: &'a str,
@@ -173,10 +183,12 @@ pub struct StateMachine<'a, B, H, N, T, Exec> where H: Hasher<Out=H256>, B: Back
extensions: Extensions,
changes_trie_storage: Option<&'a T>,
_marker: PhantomData<(H, N)>,
storage_transaction_cache: Option<&'a mut StorageTransactionCache<B::Transaction, H, N>>,
}
impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
H: Hasher<Out=H256>,
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
Exec: CodeExecutor,
B: Backend<H>,
T: ChangesTrieStorage<H, N>,
@@ -201,51 +213,61 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
overlay,
changes_trie_storage,
_marker: PhantomData,
storage_transaction_cache: None,
}
}
/// Use given `cache` as storage transaction cache.
///
/// The cache will be used to cache storage transactions that can be build while executing a
/// function in the runtime. For example, when calculating the storage root a transaction is
/// build that will be cached.
pub fn with_storage_transaction_cache(
mut self,
cache: Option<&'a mut StorageTransactionCache<B::Transaction, H, N>>,
) -> Self {
self.storage_transaction_cache = cache;
self
}
/// Execute a call using the given state backend, overlayed changes, and call executor.
/// Produces a state-backend-specific "transaction" which can be used to apply the changes
/// to the backing store, such as the disk.
///
/// On an error, no prospective changes are written to the overlay.
///
/// Note: changes to code will be in place if this call is made again. For running partial
/// blocks (e.g. a transaction at a time), ensure a different method is used.
pub fn execute(&mut self, strategy: ExecutionStrategy) -> Result<
(Vec<u8>, (B::Transaction, H::Out), Option<ChangesTrieTransaction<H, N>>),
Box<dyn Error>,
> {
///
/// Returns the SCALE encoded result of the executed function.
pub fn execute(&mut self, strategy: ExecutionStrategy) -> Result<Vec<u8>, Box<dyn Error>> {
// We are not giving a native call and thus we are sure that the result can never be a native
// value.
self.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
strategy.get_manager(),
true,
None,
)
.map(|(result, storage_tx, changes_tx)| (
result.into_encoded(),
storage_tx.expect("storage_tx is always computed when compute_tx is true; qed"),
changes_tx,
))
).map(NativeOrEncoded::into_encoded)
}
fn execute_aux<R, NC>(
&mut self,
compute_tx: bool,
use_native: bool,
native_call: Option<NC>,
) -> (
CallResult<R, Exec::Error>,
bool,
Option<(B::Transaction, H::Out)>,
Option<ChangesTrieTransaction<H, N>>,
) where
R: Decode + Encode + PartialEq,
NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
{
let mut cache = StorageTransactionCache::default();
let cache = match self.storage_transaction_cache.as_mut() {
Some(cache) => cache,
None => &mut cache,
};
let mut ext = Ext::new(
self.overlay,
cache,
self.backend,
self.changes_trie_storage.clone(),
Some(&mut self.extensions),
@@ -268,13 +290,6 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
native_call,
);
let (storage_delta, changes_delta) = if compute_tx {
let (storage_delta, changes_delta) = ext.transaction();
(Some(storage_delta), changes_delta)
} else {
(None, None)
};
trace!(
target: "state-trace", "{:04x}: Return. Native={:?}, Result={:?}",
id,
@@ -282,37 +297,28 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
result,
);
(result, was_native, storage_delta, changes_delta)
(result, was_native)
}
fn execute_call_with_both_strategy<Handler, R, NC>(
&mut self,
compute_tx: bool,
mut native_call: Option<NC>,
orig_prospective: OverlayedChangeSet,
on_consensus_failure: Handler,
) -> (
CallResult<R, Exec::Error>,
Option<(B::Transaction, H::Out)>,
Option<ChangesTrieTransaction<H, N>>,
) where
R: Decode + Encode + PartialEq,
NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
Handler: FnOnce(
CallResult<R, Exec::Error>,
CallResult<R, Exec::Error>,
) -> CallResult<R, Exec::Error>
) -> CallResult<R, Exec::Error>
where
R: Decode + Encode + PartialEq,
NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
Handler: FnOnce(
CallResult<R, Exec::Error>,
CallResult<R, Exec::Error>,
) -> CallResult<R, Exec::Error>
{
let (result, was_native, storage_delta, changes_delta) = self.execute_aux(
compute_tx,
true,
native_call.take(),
);
let (result, was_native) = self.execute_aux(true, native_call.take());
if was_native {
self.overlay.prospective = orig_prospective.clone();
let (wasm_result, _, wasm_storage_delta, wasm_changes_delta) = self.execute_aux(
compute_tx,
let (wasm_result, _) = self.execute_aux(
false,
native_call,
);
@@ -321,71 +327,62 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
&& result.as_ref().ok() == wasm_result.as_ref().ok())
|| result.is_err() && wasm_result.is_err()
{
(result, storage_delta, changes_delta)
result
} else {
(on_consensus_failure(wasm_result, result), wasm_storage_delta, wasm_changes_delta)
on_consensus_failure(wasm_result, result)
}
} else {
(result, storage_delta, changes_delta)
result
}
}
fn execute_call_with_native_else_wasm_strategy<R, NC>(
&mut self,
compute_tx: bool,
mut native_call: Option<NC>,
orig_prospective: OverlayedChangeSet,
) -> (
CallResult<R, Exec::Error>,
Option<(B::Transaction, H::Out)>,
Option<ChangesTrieTransaction<H, N>>,
) where
R: Decode + Encode + PartialEq,
NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
) -> CallResult<R, Exec::Error>
where
R: Decode + Encode + PartialEq,
NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
{
let (result, was_native, storage_delta, changes_delta) = self.execute_aux(
compute_tx,
let (result, was_native) = self.execute_aux(
true,
native_call.take(),
);
if !was_native || result.is_ok() {
(result, storage_delta, changes_delta)
result
} else {
self.overlay.prospective = orig_prospective.clone();
let (wasm_result, _, wasm_storage_delta, wasm_changes_delta) = self.execute_aux(
compute_tx,
let (wasm_result, _) = self.execute_aux(
false,
native_call,
);
(wasm_result, wasm_storage_delta, wasm_changes_delta)
wasm_result
}
}
/// Execute a call using the given state backend, overlayed changes, and call executor.
/// Produces a state-backend-specific "transaction" which can be used to apply the changes
/// to the backing store, such as the disk.
///
/// On an error, no prospective changes are written to the overlay.
///
/// Note: changes to code will be in place if this call is made again. For running partial
/// blocks (e.g. a transaction at a time), ensure a different method is used.
///
/// Returns the result of the executed function either in native reprensentation `R` or
/// in SCALE encoded representation.
pub fn execute_using_consensus_failure_handler<Handler, R, NC>(
&mut self,
manager: ExecutionManager<Handler>,
compute_tx: bool,
mut native_call: Option<NC>,
) -> Result<(
NativeOrEncoded<R>,
Option<(B::Transaction, H::Out)>,
Option<ChangesTrieTransaction<H, N>>,
), Box<dyn Error>> where
R: Decode + Encode + PartialEq,
NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
Handler: FnOnce(
CallResult<R, Exec::Error>,
CallResult<R, Exec::Error>,
) -> CallResult<R, Exec::Error>
) -> Result<NativeOrEncoded<R>, Box<dyn Error>>
where
R: Decode + Encode + PartialEq,
NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
Handler: FnOnce(
CallResult<R, Exec::Error>,
CallResult<R, Exec::Error>,
) -> CallResult<R, Exec::Error>
{
// read changes trie configuration. The reason why we're doing it here instead of the
// `OverlayedChanges` constructor is that we need proofs for this read as a part of
@@ -405,10 +402,9 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
let result = {
let orig_prospective = self.overlay.prospective.clone();
let (result, storage_delta, changes_delta) = match manager {
match manager {
ExecutionManager::Both(on_consensus_failure) => {
self.execute_call_with_both_strategy(
compute_tx,
native_call.take(),
orig_prospective,
on_consensus_failure,
@@ -416,7 +412,6 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
},
ExecutionManager::NativeElseWasm => {
self.execute_call_with_native_else_wasm_strategy(
compute_tx,
native_call.take(),
orig_prospective,
)
@@ -426,15 +421,12 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
BackendTrustLevel::Trusted => None,
BackendTrustLevel::Untrusted => Some(sp_panic_handler::AbortGuard::never_abort()),
};
let res = self.execute_aux(compute_tx, false, native_call);
(res.0, res.2, res.3)
self.execute_aux(false, native_call).0
},
ExecutionManager::NativeWhenPossible => {
let res = self.execute_aux(compute_tx, true, native_call);
(res.0, res.2, res.3)
self.execute_aux(true, native_call).0
},
};
result.map(move |out| (out, storage_delta, changes_delta))
}
};
if result.is_ok() {
@@ -455,7 +447,8 @@ pub fn prove_execution<B, H, Exec>(
) -> Result<(Vec<u8>, StorageProof), Box<dyn Error>>
where
B: Backend<H>,
H: Hasher<Out=H256>,
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
Exec: CodeExecutor,
{
let trie_backend = backend.as_trie_backend()
@@ -481,7 +474,8 @@ pub fn prove_execution_on_trie_backend<S, H, Exec>(
) -> Result<(Vec<u8>, StorageProof), Box<dyn Error>>
where
S: trie_backend_essence::TrieBackendStorage<H>,
H: Hasher<Out=H256>,
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
Exec: CodeExecutor,
{
let proving_backend = proving_backend::ProvingBackend::new(trie_backend);
@@ -489,9 +483,8 @@ where
&proving_backend, None, overlay, exec, method, call_data, Extensions::default(),
);
let (result, _, _) = sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
let result = sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
always_wasm(),
false,
None,
)?;
let proof = sm.backend.extract_proof();
@@ -508,9 +501,9 @@ pub fn execution_proof_check<H, Exec>(
call_data: &[u8],
) -> Result<Vec<u8>, Box<dyn Error>>
where
H: Hasher<Out=H256>,
H: Hasher,
Exec: CodeExecutor,
H::Out: Ord + 'static,
H::Out: Ord + 'static + codec::Codec,
{
let trie_backend = create_proof_check_backend::<H>(root.into(), proof)?;
execution_proof_check_on_trie_backend(&trie_backend, overlay, exec, method, call_data)
@@ -525,7 +518,8 @@ pub fn execution_proof_check_on_trie_backend<H, Exec>(
call_data: &[u8],
) -> Result<Vec<u8>, Box<dyn Error>>
where
H: Hasher<Out=H256>,
H: Hasher,
H::Out: Ord + 'static + codec::Codec,
Exec: CodeExecutor,
{
let mut sm = StateMachine::<_, H, _, InMemoryChangesTrieStorage<H, u64>, Exec>::new(
@@ -534,9 +528,8 @@ where
sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
always_untrusted_wasm(),
false,
None,
).map(|(result, _, _)| result.into_encoded())
).map(NativeOrEncoded::into_encoded)
}
/// Generate storage read proof.
@@ -546,8 +539,8 @@ pub fn prove_read<B, H, I>(
) -> Result<StorageProof, Box<dyn Error>>
where
B: Backend<H>,
H: Hasher<Out=H256>,
H::Out: Ord,
H: Hasher,
H::Out: Ord + Codec,
I: IntoIterator,
I::Item: AsRef<[u8]>,
{
@@ -739,7 +732,6 @@ mod tests {
use codec::Encode;
use overlayed_changes::OverlayedValue;
use super::*;
use super::backend::InMemory;
use super::ext::Ext;
use super::changes_trie::{
InMemoryStorage as InMemoryChangesTrieStorage,
@@ -825,7 +817,7 @@ mod tests {
);
assert_eq!(
state_machine.execute(ExecutionStrategy::NativeWhenPossible).unwrap().0,
state_machine.execute(ExecutionStrategy::NativeWhenPossible).unwrap(),
vec![66],
);
}
@@ -852,7 +844,7 @@ mod tests {
Default::default(),
);
assert_eq!(state_machine.execute(ExecutionStrategy::NativeElseWasm).unwrap().0, vec![66]);
assert_eq!(state_machine.execute(ExecutionStrategy::NativeElseWasm).unwrap(), vec![66]);
}
#[test]
@@ -883,7 +875,6 @@ mod tests {
consensus_failed = true;
we
}),
true,
None,
).is_err()
);
@@ -933,7 +924,7 @@ mod tests {
b"abc".to_vec() => b"2".to_vec(),
b"bbb".to_vec() => b"3".to_vec()
];
let mut state = InMemory::<Blake2Hasher>::from(initial);
let mut state = InMemoryBackend::<Blake2Hasher>::from(initial);
let backend = state.as_trie_backend().unwrap();
let mut overlay = OverlayedChanges {
committed: map![
@@ -949,8 +940,10 @@ mod tests {
{
let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
let mut cache = StorageTransactionCache::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
backend,
Some(&changes_trie_storage),
None,
@@ -975,12 +968,14 @@ mod tests {
#[test]
fn set_child_storage_works() {
let mut state = InMemory::<Blake2Hasher>::default();
let mut state = InMemoryBackend::<Blake2Hasher>::default();
let backend = state.as_trie_backend().unwrap();
let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
let mut overlay = OverlayedChanges::default();
let mut cache = StorageTransactionCache::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
backend,
Some(&changes_trie_storage),
None,
@@ -1104,8 +1099,10 @@ mod tests {
let mut transaction = {
let backend = test_trie();
let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
let mut cache = StorageTransactionCache::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
&backend,
Some(&changes_trie_storage),
None,
@@ -1113,7 +1110,7 @@ mod tests {
ext.set_child_storage(subtrie1, CHILD_INFO_1, b"abc".to_vec(), b"def".to_vec());
ext.set_child_storage(subtrie2, CHILD_INFO_2, b"abc".to_vec(), b"def".to_vec());
ext.storage_root();
(ext.transaction().0).0
cache.transaction.unwrap()
};
let mut duplicate = false;
for (k, (value, rc)) in transaction.drain().iter() {
@@ -16,14 +16,23 @@
//! The overlayed changes to state.
use crate::{
backend::Backend, ChangesTrieTransaction,
changes_trie::{
NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig, BlockNumber, build_changes_trie,
Storage as ChangesTrieStorage,
},
};
#[cfg(test)]
use std::iter::FromIterator;
use std::collections::{HashMap, BTreeMap, BTreeSet};
use codec::Decode;
use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig};
use codec::{Decode, Encode};
use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, OwnedChildInfo, ChildInfo};
use std::{mem, ops};
use hash_db::Hasher;
/// The overlayed changes to state to be queried on top of the backend.
///
/// A transaction shares all prospective changes within an inner overlay
@@ -60,6 +69,92 @@ pub struct OverlayedChangeSet {
pub children: HashMap<Vec<u8>, (BTreeMap<Vec<u8>, OverlayedValue>, OwnedChildInfo)>,
}
/// A storage changes structure that can be generated by the data collected in [`OverlayedChanges`].
///
/// This contains all the changes to the storage and transactions to apply theses changes to the
/// backend.
pub struct StorageChanges<Transaction, H: Hasher, N: BlockNumber> {
/// All changes to the main storage.
///
/// A value of `None` means that it was deleted.
pub main_storage_changes: Vec<(Vec<u8>, Option<Vec<u8>>)>,
/// All changes to the child storages.
pub child_storage_changes: Vec<(Vec<u8>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>,
/// A transaction for the backend that contains all changes from
/// [`main_storage_changes`](Self::main_storage_changes) and from
/// [`child_storage_changes`](Self::child_storage_changes).
pub transaction: Transaction,
/// The storage root after applying the transaction.
pub transaction_storage_root: H::Out,
/// Contains the transaction for the backend for the changes trie.
///
/// If changes trie is disabled the value is set to `None`.
pub changes_trie_transaction: Option<ChangesTrieTransaction<H, N>>,
}
impl<Transaction, H: Hasher, N: BlockNumber> StorageChanges<Transaction, H, N> {
/// Deconstruct into the inner values
pub fn into_inner(self) -> (
Vec<(Vec<u8>, Option<Vec<u8>>)>,
Vec<(Vec<u8>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>,
Transaction,
H::Out,
Option<ChangesTrieTransaction<H, N>>,
) {
(
self.main_storage_changes,
self.child_storage_changes,
self.transaction,
self.transaction_storage_root,
self.changes_trie_transaction,
)
}
}
/// The storage transaction are calculated as part of the `storage_root` and
/// `changes_trie_storage_root`. These transactions can be reused for importing the block into the
/// storage. So, we cache them to not require a recomputation of those transactions.
pub struct StorageTransactionCache<Transaction, H: Hasher, N: BlockNumber> {
/// Contains the changes for the main and the child storages as one transaction.
pub(crate) transaction: Option<Transaction>,
/// The storage root after applying the transaction.
pub(crate) transaction_storage_root: Option<H::Out>,
/// Contains the changes trie transaction.
pub(crate) changes_trie_transaction: Option<Option<ChangesTrieTransaction<H, N>>>,
/// The storage root after applying the changes trie transaction.
pub(crate) changes_trie_transaction_storage_root: Option<Option<H::Out>>,
}
impl<Transaction, H: Hasher, N: BlockNumber> StorageTransactionCache<Transaction, H, N> {
/// Reset the cached transactions.
pub fn reset(&mut self) {
*self = Self::default();
}
}
impl<Transaction, H: Hasher, N: BlockNumber> Default for StorageTransactionCache<Transaction, H, N> {
fn default() -> Self {
Self {
transaction: None,
transaction_storage_root: None,
changes_trie_transaction: None,
changes_trie_transaction_storage_root: None,
}
}
}
impl<Transaction: Default, H: Hasher, N: BlockNumber> Default for StorageChanges<Transaction, H, N> {
fn default() -> Self {
Self {
main_storage_changes: Default::default(),
child_storage_changes: Default::default(),
transaction: Default::default(),
transaction_storage_root: Default::default(),
changes_trie_transaction: None,
}
}
}
#[cfg(test)]
impl FromIterator<(Vec<u8>, OverlayedValue)> for OverlayedChangeSet {
fn from_iter<T: IntoIterator<Item = (Vec<u8>, OverlayedValue)>>(iter: T) -> Self {
@@ -105,7 +200,7 @@ impl OverlayedChanges {
true
}
/// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered
/// Returns a double-Option: None if the key is unknown (i.e. and the query should be referred
/// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose
/// value has been set.
pub fn storage(&self, key: &[u8]) -> Option<Option<&[u8]>> {
@@ -114,7 +209,7 @@ impl OverlayedChanges {
.map(|x| x.value.as_ref().map(AsRef::as_ref))
}
/// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered
/// Returns a double-Option: None if the key is unknown (i.e. and the query should be referred
/// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose
/// value has been set.
pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option<Option<&[u8]>> {
@@ -237,7 +332,7 @@ impl OverlayedChanges {
}
}
// Then do the same with keys from commited changes.
// Then do the same with keys from committed changes.
// NOTE that we are making changes in the prospective change set.
for key in self.committed.top.keys() {
if key.starts_with(prefix) {
@@ -338,15 +433,61 @@ impl OverlayedChanges {
impl Iterator<Item=(Vec<u8>, (impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)>, OwnedChildInfo))>,
){
assert!(self.prospective.is_empty());
(self.committed.top.into_iter().map(|(k, v)| (k, v.value)),
(
self.committed.top.into_iter().map(|(k, v)| (k, v.value)),
self.committed.children.into_iter()
.map(|(sk, (v, ci))| (sk, (v.into_iter().map(|(k, v)| (k, v.value)), ci))))
.map(|(sk, (v, ci))| (sk, (v.into_iter().map(|(k, v)| (k, v.value)), ci)))
)
}
/// Convert this instance with all changes into a [`StorageChanges`] instance.
pub fn into_storage_changes<
B: Backend<H>, H: Hasher, N: BlockNumber, T: ChangesTrieStorage<H, N>
>(
self,
backend: &B,
changes_trie_storage: Option<&T>,
parent_hash: H::Out,
mut cache: StorageTransactionCache<B::Transaction, H, N>,
) -> Result<StorageChanges<B::Transaction, H, N>, String> where H::Out: Ord + Encode + 'static {
// If the transaction does not exist, we generate it.
if cache.transaction.is_none() {
self.storage_root(backend, &mut cache);
}
let (transaction, transaction_storage_root) = cache.transaction.take()
.and_then(|t| cache.transaction_storage_root.take().map(|tr| (t, tr)))
.expect("Transaction was be generated as part of `storage_root`; qed");
// If the transaction does not exist, we generate it.
if cache.changes_trie_transaction.is_none() {
self.changes_trie_root(
backend,
changes_trie_storage,
parent_hash,
false,
&mut cache,
).map_err(|_| "Failed to generate changes trie transaction")?;
}
let changes_trie_transaction = cache.changes_trie_transaction
.take()
.expect("Changes trie transaction was generated by `changes_trie_root`; qed");
let (main_storage_changes, child_storage_changes) = self.into_committed();
Ok(StorageChanges {
main_storage_changes: main_storage_changes.collect(),
child_storage_changes: child_storage_changes.map(|(sk, it)| (sk, it.0.collect())).collect(),
transaction,
transaction_storage_root,
changes_trie_transaction,
})
}
/// Inserts storage entry responsible for current extrinsic index.
#[cfg(test)]
pub(crate) fn set_extrinsic_index(&mut self, extrinsic_index: u32) {
use codec::Encode;
self.prospective.top.insert(EXTRINSIC_INDEX.to_vec(), OverlayedValue {
value: Some(extrinsic_index.encode()),
extrinsics: None,
@@ -356,7 +497,7 @@ impl OverlayedChanges {
/// Returns current extrinsic index to use in changes trie construction.
/// None is returned if it is not set or changes trie config is not set.
/// Persistent value (from the backend) can be ignored because runtime must
/// set this index before first and unset after last extrinsic is executied.
/// set this index before first and unset after last extrinsic is executed.
/// Changes that are made outside of extrinsics, are marked with
/// `NO_EXTRINSIC_INDEX` index.
fn extrinsic_index(&self) -> Option<u32> {
@@ -369,6 +510,75 @@ impl OverlayedChanges {
}
}
/// Generate the storage root using `backend` and all changes from `prospective` and `committed`.
///
/// Returns the storage root and caches storage transaction in the given `cache`.
pub fn storage_root<H: Hasher, N: BlockNumber, B: Backend<H>>(
&self,
backend: &B,
cache: &mut StorageTransactionCache<B::Transaction, H, N>,
) -> H::Out
where H::Out: Ord + Encode,
{
let child_storage_keys = self.prospective.children.keys()
.chain(self.committed.children.keys());
let child_delta_iter = child_storage_keys.map(|storage_key|
(
storage_key.clone(),
self.committed.children.get(storage_key)
.into_iter()
.flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))
.chain(
self.prospective.children.get(storage_key)
.into_iter()
.flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))
),
self.child_info(storage_key).cloned()
.expect("child info initialized in either committed or prospective"),
)
);
// compute and memoize
let delta = self.committed.top.iter().map(|(k, v)| (k.clone(), v.value.clone()))
.chain(self.prospective.top.iter().map(|(k, v)| (k.clone(), v.value.clone())));
let (root, transaction) = backend.full_storage_root(delta, child_delta_iter);
cache.transaction = Some(transaction);
cache.transaction_storage_root = Some(root);
root
}
/// Generate the changes trie root.
///
/// Returns the changes trie root and caches the storage transaction into the given `cache`.
///
/// # Panics
///
/// Panics on storage error, when `panic_on_storage_error` is set.
pub fn changes_trie_root<H: Hasher, N: BlockNumber, B: Backend<H>, T: ChangesTrieStorage<H, N>>(
&self,
backend: &B,
changes_trie_storage: Option<&T>,
parent_hash: H::Out,
panic_on_storage_error: bool,
cache: &mut StorageTransactionCache<B::Transaction, H, N>,
) -> Result<Option<H::Out>, ()> where H::Out: Ord + Encode + 'static {
build_changes_trie::<_, T, H, N>(
backend,
changes_trie_storage,
self,
parent_hash,
panic_on_storage_error,
).map(|r| {
let root = r.as_ref().map(|r| r.1).clone();
cache.changes_trie_transaction = Some(r.map(|(db, _, cache)| (db, cache)));
cache.changes_trie_transaction_storage_root = Some(root);
root
})
}
/// Get child info for a storage key.
/// Take the latest value so prospective first.
pub fn child_info(&self, storage_key: &[u8]) -> Option<&OwnedChildInfo> {
@@ -445,7 +655,7 @@ mod tests {
use sp_core::{
Blake2Hasher, traits::Externalities, storage::well_known_keys::EXTRINSIC_INDEX,
};
use crate::backend::InMemory;
use crate::InMemoryBackend;
use crate::changes_trie::InMemoryStorage as InMemoryChangesTrieStorage;
use crate::ext::Ext;
use super::*;
@@ -494,7 +704,7 @@ mod tests {
(b"dogglesworth".to_vec(), b"catXXX".to_vec()),
(b"doug".to_vec(), b"notadog".to_vec()),
].into_iter().collect();
let backend = InMemory::<Blake2Hasher>::from(initial);
let backend = InMemoryBackend::<Blake2Hasher>::from(initial);
let mut overlay = OverlayedChanges {
committed: vec![
(b"dog".to_vec(), Some(b"puppy".to_vec()).into()),
@@ -509,8 +719,10 @@ mod tests {
};
let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
let mut cache = StorageTransactionCache::default();
let mut ext = Ext::new(
&mut overlay,
&mut cache,
&backend,
Some(&changes_trie_storage),
None,
@@ -22,8 +22,8 @@ use codec::{Decode, Encode, Codec};
use log::debug;
use hash_db::{Hasher, HashDB, EMPTY_PREFIX, Prefix};
use sp_trie::{
MemoryDB, PrefixedMemoryDB, default_child_trie_root,
read_trie_value_with, read_child_trie_value_with, record_all_keys
MemoryDB, default_child_trie_root, read_trie_value_with, read_child_trie_value_with,
record_all_keys
};
pub use sp_trie::Recorder;
pub use sp_trie::trie_types::{Layout, TrieError};
@@ -136,7 +136,7 @@ impl<'a, S, H> ProvingBackendRecorder<'a, S, H>
&eph,
self.backend.root(),
key,
&mut *self.proof_recorder
&mut *self.proof_recorder,
).map_err(map_e)
}
@@ -238,7 +238,9 @@ impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> ProvingBackend<'a, S, H>
}
}
impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> TrieBackendStorage<H> for ProofRecorderBackend<'a, S, H> {
impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> TrieBackendStorage<H>
for ProofRecorderBackend<'a, S, H>
{
type Overlay = S::Overlay;
fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
@@ -251,7 +253,9 @@ impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> TrieBackendStorage<H> fo
}
}
impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> std::fmt::Debug for ProvingBackend<'a, S, H> {
impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> std::fmt::Debug
for ProvingBackend<'a, S, H>
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "ProvingBackend")
}
@@ -265,7 +269,7 @@ impl<'a, S, H> Backend<H> for ProvingBackend<'a, S, H>
{
type Error = String;
type Transaction = S::Overlay;
type TrieBackendStorage = PrefixedMemoryDB<H>;
type TrieBackendStorage = S;
fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
self.0.storage(key)
@@ -391,11 +395,12 @@ where
#[cfg(test)]
mod tests {
use crate::backend::{InMemory};
use crate::InMemoryBackend;
use crate::trie_backend::tests::test_trie;
use super::*;
use sp_core::{Blake2Hasher, storage::ChildStorageKey};
use crate::proving_backend::create_proof_check_backend;
use sp_trie::PrefixedMemoryDB;
const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1");
const CHILD_INFO_2: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_2");
@@ -446,7 +451,7 @@ mod tests {
#[test]
fn proof_recorded_and_checked() {
let contents = (0..64).map(|i| (vec![i], Some(vec![i]))).collect::<Vec<_>>();
let in_memory = InMemory::<Blake2Hasher>::default();
let in_memory = InMemoryBackend::<Blake2Hasher>::default();
let mut in_memory = in_memory.update(vec![(None, contents)]);
let in_memory_root = in_memory.storage_root(::std::iter::empty()).0;
(0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i]));
@@ -478,7 +483,7 @@ mod tests {
(Some((own2.clone(), CHILD_INFO_2.to_owned())),
(10..15).map(|i| (vec![i], Some(vec![i]))).collect()),
];
let in_memory = InMemory::<Blake2Hasher>::default();
let in_memory = InMemoryBackend::<Blake2Hasher>::default();
let mut in_memory = in_memory.update(contents);
let in_memory_root = in_memory.full_storage_root::<_, Vec<_>, _>(
::std::iter::empty(),
@@ -533,5 +538,4 @@ mod tests {
vec![64]
);
}
}
@@ -19,36 +19,45 @@
use std::any::{Any, TypeId};
use hash_db::Hasher;
use crate::{
backend::{InMemory, Backend}, OverlayedChanges,
backend::Backend, OverlayedChanges, StorageTransactionCache, ext::Ext, InMemoryBackend,
changes_trie::{
InMemoryStorage as ChangesTrieInMemoryStorage,
BlockNumber as ChangesTrieBlockNumber,
},
ext::Ext,
};
use sp_core::{
storage::{
well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES, is_child_storage_key},
Storage,
},
hash::H256, Blake2Hasher,
Blake2Hasher,
};
use codec::Encode;
use sp_externalities::{Extensions, Extension};
/// Simple HashMap-based Externalities impl.
pub struct TestExternalities<H: Hasher<Out=H256>=Blake2Hasher, N: ChangesTrieBlockNumber=u64> {
pub struct TestExternalities<H: Hasher = Blake2Hasher, N: ChangesTrieBlockNumber = u64>
where
H::Out: codec::Codec,
{
overlay: OverlayedChanges,
backend: InMemory<H>,
storage_transaction_cache: StorageTransactionCache<
<InMemoryBackend<H> as Backend<H>>::Transaction, H, N
>,
backend: InMemoryBackend<H>,
changes_trie_storage: ChangesTrieInMemoryStorage<H, N>,
extensions: Extensions,
}
impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> TestExternalities<H, N> {
impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
where
H::Out: Ord + 'static + codec::Codec
{
/// Get externalities implementation.
pub fn ext(&mut self) -> Ext<H, N, InMemory<H>, ChangesTrieInMemoryStorage<H, N>> {
pub fn ext(&mut self) -> Ext<H, N, InMemoryBackend<H>, ChangesTrieInMemoryStorage<H, N>> {
Ext::new(
&mut self.overlay,
&mut self.storage_transaction_cache,
&self.backend,
Some(&self.changes_trie_storage),
Some(&mut self.extensions),
@@ -81,6 +90,7 @@ impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> TestExternalities<H, N> {
changes_trie_storage: ChangesTrieInMemoryStorage::new(),
backend: storage.into(),
extensions: Default::default(),
storage_transaction_cache: Default::default(),
}
}
@@ -100,7 +110,7 @@ impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> TestExternalities<H, N> {
}
/// Return a new backend with all pending value.
pub fn commit_all(&self) -> InMemory<H> {
pub fn commit_all(&self) -> InMemoryBackend<H> {
let top: Vec<_> = self.overlay.committed.top.clone().into_iter()
.chain(self.overlay.prospective.top.clone().into_iter())
.map(|(k, v)| (k, v.value)).collect();
@@ -129,13 +139,18 @@ impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> TestExternalities<H, N> {
}
}
impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> std::fmt::Debug for TestExternalities<H, N> {
fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
impl<H: Hasher, N: ChangesTrieBlockNumber> std::fmt::Debug for TestExternalities<H, N>
where H::Out: codec::Codec,
{
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(f, "overlay: {:?}\nbackend: {:?}", self.overlay, self.backend.pairs())
}
}
impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> PartialEq for TestExternalities<H, N> {
impl<H: Hasher, N: ChangesTrieBlockNumber> PartialEq for TestExternalities<H, N>
where
H::Out: Ord + 'static + codec::Codec
{
/// This doesn't test if they are in the same state, only if they contains the
/// same data at this state
fn eq(&self, other: &TestExternalities<H, N>) -> bool {
@@ -143,18 +158,25 @@ impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> PartialEq for TestExternali
}
}
impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> Default for TestExternalities<H, N> {
impl<H: Hasher, N: ChangesTrieBlockNumber> Default for TestExternalities<H, N>
where
H::Out: Ord + 'static + codec::Codec,
{
fn default() -> Self { Self::new(Default::default()) }
}
impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> From<Storage> for TestExternalities<H, N> {
impl<H: Hasher, N: ChangesTrieBlockNumber> From<Storage> for TestExternalities<H, N>
where
H::Out: Ord + 'static + codec::Codec,
{
fn from(storage: Storage) -> Self {
Self::new(storage)
}
}
impl<H, N> sp_externalities::ExtensionStore for TestExternalities<H, N> where
H: Hasher<Out=H256>,
H: Hasher,
H::Out: codec::Codec,
N: ChangesTrieBlockNumber,
{
fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any> {
@@ -237,7 +237,6 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> where H::Out:
self.keys_values_with_prefix_inner(&self.root, prefix, |k, _v| f(k), None)
}
fn keys_values_with_prefix_inner<F: FnMut(&[u8], &[u8])>(
&self,
root: &H::Out,
@@ -285,7 +284,6 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> where H::Out:
pub fn for_key_values_with_prefix<F: FnMut(&[u8], &[u8])>(&self, prefix: &[u8], f: F) {
self.keys_values_with_prefix_inner(&self.root, prefix, f, None)
}
}
pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> {
@@ -293,20 +291,16 @@ pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> {
overlay: &'a mut S::Overlay,
}
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: 'a + Hasher
> hash_db::AsPlainDB<H::Out, DBValue>
impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> hash_db::AsPlainDB<H::Out, DBValue>
for Ephemeral<'a, S, H>
{
fn as_plain_db<'b>(&'b self) -> &'b (dyn hash_db::PlainDB<H::Out, DBValue> + 'b) { self }
fn as_plain_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::PlainDB<H::Out, DBValue> + 'b) { self }
fn as_plain_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::PlainDB<H::Out, DBValue> + 'b) {
self
}
}
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: 'a + Hasher
> hash_db::AsHashDB<H, DBValue>
impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> hash_db::AsHashDB<H, DBValue>
for Ephemeral<'a, S, H>
{
fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB<H, DBValue> + 'b) { self }
@@ -322,10 +316,7 @@ impl<'a, S: TrieBackendStorage<H>, H: Hasher> Ephemeral<'a, S, H> {
}
}
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: Hasher
> hash_db::PlainDB<H::Out, DBValue>
impl<'a, S: 'a + TrieBackendStorage<H>, H: Hasher> hash_db::PlainDB<H::Out, DBValue>
for Ephemeral<'a, S, H>
{
fn get(&self, key: &H::Out) -> Option<DBValue> {
@@ -355,20 +346,14 @@ impl<'a,
}
}
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: Hasher
> hash_db::PlainDBRef<H::Out, DBValue>
impl<'a, S: 'a + TrieBackendStorage<H>, H: Hasher> hash_db::PlainDBRef<H::Out, DBValue>
for Ephemeral<'a, S, H>
{
fn get(&self, key: &H::Out) -> Option<DBValue> { hash_db::PlainDB::get(self, key) }
fn contains(&self, key: &H::Out) -> bool { hash_db::PlainDB::contains(self, key) }
}
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: Hasher
> hash_db::HashDB<H, DBValue>
impl<'a, S: 'a + TrieBackendStorage<H>, H: Hasher> hash_db::HashDB<H, DBValue>
for Ephemeral<'a, S, H>
{
fn get(&self, key: &H::Out, prefix: Prefix) -> Option<DBValue> {
@@ -402,14 +387,16 @@ impl<'a,
}
}
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: Hasher
> hash_db::HashDBRef<H, DBValue>
impl<'a, S: 'a + TrieBackendStorage<H>, H: Hasher> hash_db::HashDBRef<H, DBValue>
for Ephemeral<'a, S, H>
{
fn get(&self, key: &H::Out, prefix: Prefix) -> Option<DBValue> { hash_db::HashDB::get(self, key, prefix) }
fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { hash_db::HashDB::contains(self, key, prefix) }
fn get(&self, key: &H::Out, prefix: Prefix) -> Option<DBValue> {
hash_db::HashDB::get(self, key, prefix)
}
fn contains(&self, key: &H::Out, prefix: Prefix) -> bool {
hash_db::HashDB::contains(self, key, prefix)
}
}
/// Key-value pairs storage that is used by trie backend essence.