Switch to shiny new fast, RLP-less trie (#795)

* Bump codec

* Fix tests

* Patricia trie builds

* Introduce trie

* Some yak shaving.

* Some fixes

* Remove RLP ref

* Fixes

* It builds!

* Some tests fixed

* Another test fix

* Rejig more hashes

* substrate-trie::iterator_works test

* Update lock

* Polish

* Docs

* Undo incorrect "fix" for tests

* Fix nits
This commit is contained in:
Gav Wood
2018-09-25 15:32:22 +01:00
committed by Arkadiy Paronyan
parent b02c274374
commit 82d6ca3484
90 changed files with 1977 additions and 1129 deletions
+25 -27
View File
@@ -20,19 +20,17 @@ use std::{error, fmt};
use std::cmp::Ord;
use std::collections::HashMap;
use std::marker::PhantomData;
use hashdb::Hasher;
use memorydb::MemoryDB;
use rlp::Encodable;
use hash_db::Hasher;
use trie_backend::TrieBackend;
use trie_backend_essence::TrieBackendStorage;
use patricia_trie::{TrieDBMut, TrieMut, NodeCodec};
use substrate_trie::{TrieDBMut, TrieMut, MemoryDB, trie_root};
use heapsize::HeapSizeOf;
/// A state backend is used to read state data and can have changes committed
/// to it.
///
/// The clone operation (if implemented) should be cheap.
pub trait Backend<H: Hasher, C: NodeCodec<H>> {
pub trait Backend<H: Hasher> {
/// An error type when fetching data is not possible.
type Error: super::Error;
@@ -59,13 +57,13 @@ pub trait Backend<H: Hasher, C: NodeCodec<H>> {
fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
H::Out: Ord + Encodable;
H::Out: Ord;
/// Get all key/value pairs into a Vec.
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)>;
/// Try convert into trie backend.
fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H, C>>;
fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H>>;
}
/// Error impossible.
@@ -86,39 +84,38 @@ impl error::Error for Void {
/// In-memory backend. Fully recomputes tries on each commit but useful for
/// tests.
#[derive(Eq)]
pub struct InMemory<H, C> {
pub struct InMemory<H> {
inner: HashMap<Vec<u8>, Vec<u8>>,
_hasher: PhantomData<H>,
_codec: PhantomData<C>,
}
impl<H, C> Default for InMemory<H, C> {
impl<H> Default for InMemory<H> {
fn default() -> Self {
InMemory {
inner: Default::default(),
_hasher: PhantomData,
_codec: PhantomData,
}
}
}
impl<H, C> Clone for InMemory<H, C> {
impl<H> Clone for InMemory<H> {
fn clone(&self) -> Self {
InMemory {
inner: self.inner.clone(), _hasher: PhantomData, _codec: PhantomData,
inner: self.inner.clone(),
_hasher: PhantomData,
}
}
}
impl<H, C> PartialEq for InMemory<H, C> {
impl<H> PartialEq for InMemory<H> {
fn eq(&self, other: &Self) -> bool {
self.inner.eq(&other.inner)
}
}
impl<H: Hasher, C: NodeCodec<H>> InMemory<H, C> where H::Out: HeapSizeOf {
impl<H: Hasher> InMemory<H> where H::Out: HeapSizeOf {
/// Copy the state, with applied updates
pub fn update(&self, changes: <Self as Backend<H, C>>::Transaction) -> Self {
pub fn update(&self, changes: <Self as Backend<H>>::Transaction) -> Self {
let mut inner: HashMap<_, _> = self.inner.clone();
for (key, val) in changes {
match val {
@@ -131,17 +128,18 @@ impl<H: Hasher, C: NodeCodec<H>> InMemory<H, C> where H::Out: HeapSizeOf {
}
}
impl<H, C> From<HashMap<Vec<u8>, Vec<u8>>> for InMemory<H, C> {
impl<H> From<HashMap<Vec<u8>, Vec<u8>>> for InMemory<H> {
fn from(inner: HashMap<Vec<u8>, Vec<u8>>) -> Self {
InMemory {
inner: inner, _hasher: PhantomData, _codec: PhantomData
inner: inner,
_hasher: PhantomData,
}
}
}
impl super::Error for Void {}
impl<H: Hasher, C: NodeCodec<H>> Backend<H, C> for InMemory<H, C> where H::Out: HeapSizeOf {
impl<H: Hasher> Backend<H> for InMemory<H> where H::Out: HeapSizeOf {
type Error = Void;
type Transaction = Vec<(Vec<u8>, Option<Vec<u8>>)>;
type TrieBackendStorage = MemoryDB<H>;
@@ -161,12 +159,12 @@ impl<H: Hasher, C: NodeCodec<H>> Backend<H, C> for InMemory<H, C> where H::Out:
fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
<H as Hasher>::Out: Ord + Encodable,
<H as Hasher>::Out: Ord,
{
let existing_pairs = self.inner.iter().map(|(k, v)| (k.clone(), Some(v.clone())));
let transaction: Vec<_> = delta.into_iter().collect();
let root = ::triehash::trie_root::<H, _, _, _>(existing_pairs.chain(transaction.iter().cloned())
let root = trie_root::<H, _, _, _>(existing_pairs.chain(transaction.iter().cloned())
.collect::<HashMap<_, _>>()
.into_iter()
.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
@@ -179,24 +177,24 @@ impl<H: Hasher, C: NodeCodec<H>> Backend<H, C> for InMemory<H, C> where H::Out:
self.inner.iter().map(|(k, v)| (k.clone(), v.clone())).collect()
}
fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H, C>> {
let mut mdb = MemoryDB::new();
let root = insert_into_memory_db::<H, C, _>(&mut mdb, self.inner.into_iter())?;
fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H>> {
let mut mdb = MemoryDB::default(); // TODO: should be more correct and use ::new()
let root = insert_into_memory_db::<H, _>(&mut mdb, self.inner.into_iter())?;
Some(TrieBackend::new(mdb, root))
}
}
/// Insert input pairs into memory db.
pub(crate) fn insert_into_memory_db<H, C, I>(mdb: &mut MemoryDB<H>, input: I) -> Option<H::Out>
pub(crate) fn insert_into_memory_db<H, I>(mdb: &mut MemoryDB<H>, input: I) -> Option<H::Out>
where
H: Hasher,
H::Out: HeapSizeOf,
C: NodeCodec<H>,
I: Iterator<Item=(Vec<u8>, Vec<u8>)>,
{
let mut root = <H as Hasher>::Out::default();
{
let mut trie = TrieDBMut::<H, C>::new(mdb, &mut root);
let mut trie = TrieDBMut::<H>::new(mdb, &mut root);
for (key, value) in input {
if let Err(e) = trie.insert(&key, &value) {
warn!(target: "trie", "Failed to write to trie: {}", e);
@@ -18,9 +18,8 @@
use std::collections::{BTreeMap, BTreeSet};
use codec::Decode;
use hashdb::Hasher;
use hash_db::Hasher;
use heapsize::HeapSizeOf;
use patricia_trie::NodeCodec;
use backend::Backend;
use overlayed_changes::OverlayedChanges;
use trie_backend_essence::{TrieBackendStorage, TrieBackendEssence};
@@ -34,19 +33,18 @@ use changes_trie::{Configuration, Storage};
/// required data.
/// Returns Ok(None) data required to prepare input pairs is not collected
/// or storage is not provided.
pub fn prepare_input<'a, B, S, H, C>(
pub fn prepare_input<'a, B, S, H>(
backend: &B,
storage: Option<&'a S>,
changes: &OverlayedChanges,
block: u64,
) -> Result<Option<Vec<InputPair>>, String>
where
B: Backend<H, C>,
B: Backend<H>,
S: Storage<H>,
&'a S: TrieBackendStorage<H>,
H: Hasher,
H::Out: HeapSizeOf,
C: NodeCodec<H>,
{
let (storage, config) = match (storage, changes.changes_trie_config.as_ref()) {
(Some(storage), Some(config)) => (storage, config),
@@ -58,7 +56,7 @@ pub fn prepare_input<'a, B, S, H, C>(
backend,
block,
changes)?);
input.extend(prepare_digest_input::<_, H, C>(
input.extend(prepare_digest_input::<_, H>(
block,
config,
storage)?);
@@ -67,15 +65,15 @@ pub fn prepare_input<'a, B, S, H, C>(
}
/// Prepare ExtrinsicIndex input pairs.
fn prepare_extrinsics_input<B, H, C>(
fn prepare_extrinsics_input<B, H>(
backend: &B,
block: u64,
changes: &OverlayedChanges,
) -> Result<impl Iterator<Item=InputPair>, String>
where
B: Backend<H, C>,
B: Backend<H>,
H: Hasher,
C: NodeCodec<H>,
{
let mut extrinsic_map = BTreeMap::<Vec<u8>, BTreeSet<u32>>::new();
for (key, val) in changes.prospective.iter().chain(changes.committed.iter()) {
@@ -104,7 +102,7 @@ fn prepare_extrinsics_input<B, H, C>(
}
/// Prepare DigestIndex input pairs.
fn prepare_digest_input<'a, S, H, C>(
fn prepare_digest_input<'a, S, H>(
block: u64,
config: &Configuration,
storage: &'a S
@@ -114,13 +112,12 @@ fn prepare_digest_input<'a, S, H, C>(
&'a S: TrieBackendStorage<H>,
H: Hasher,
H::Out: HeapSizeOf,
C: NodeCodec<H>,
{
let mut digest_map = BTreeMap::<Vec<u8>, BTreeSet<u64>>::new();
for digest_build_block in digest_build_iterator(config, block) {
let trie_root = storage.root(digest_build_block)?;
let trie_root = trie_root.ok_or_else(|| format!("No changes trie root for block {}", digest_build_block))?;
let trie_storage = TrieBackendEssence::<_, H, C>::new(storage, trie_root);
let trie_storage = TrieBackendEssence::<_, H>::new(storage, trie_root);
let extrinsic_prefix = ExtrinsicIndex::key_neutral_prefix(digest_build_block);
trie_storage.for_keys_with_prefix(&extrinsic_prefix, |key|
@@ -147,15 +144,15 @@ fn prepare_digest_input<'a, S, H, C>(
#[cfg(test)]
mod test {
use codec::Encode;
use primitives::{Blake2Hasher, RlpCodec};
use primitives::Blake2Hasher;
use primitives::storage::well_known_keys::EXTRINSIC_INDEX;
use backend::InMemory;
use changes_trie::storage::InMemoryStorage;
use overlayed_changes::OverlayedValue;
use super::*;
fn prepare_for_build() -> (InMemory<Blake2Hasher, RlpCodec>, InMemoryStorage<Blake2Hasher>, OverlayedChanges) {
let backend: InMemory<_, _> = vec![
fn prepare_for_build() -> (InMemory<Blake2Hasher>, InMemoryStorage<Blake2Hasher>, OverlayedChanges) {
let backend: InMemory<_> = vec![
(vec![100], vec![255]),
(vec![101], vec![255]),
(vec![102], vec![255]),
@@ -163,7 +160,7 @@ mod test {
(vec![104], vec![255]),
(vec![105], vec![255]),
].into_iter().collect::<::std::collections::HashMap<_, _>>().into();
let storage = InMemoryStorage::with_inputs::<RlpCodec>(vec![
let storage = InMemoryStorage::with_inputs(vec![
(1, vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 1, key: vec![100] }, vec![1, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 1, key: vec![101] }, vec![0, 2]),
@@ -231,7 +228,7 @@ mod test {
#[test]
fn build_changes_trie_nodes_on_non_digest_block() {
let (backend, storage, changes) = prepare_for_build();
let changes_trie_nodes = prepare_input::<_, _, _, RlpCodec>(&backend, Some(&storage), &changes, 5).unwrap();
let changes_trie_nodes = prepare_input(&backend, Some(&storage), &changes, 5).unwrap();
assert_eq!(changes_trie_nodes, Some(vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![101] }, vec![1]),
@@ -242,7 +239,7 @@ mod test {
#[test]
fn build_changes_trie_nodes_on_digest_block_l1() {
let (backend, storage, changes) = prepare_for_build();
let changes_trie_nodes = prepare_input::<_, _, _, RlpCodec>(&backend, Some(&storage), &changes, 4).unwrap();
let changes_trie_nodes = prepare_input(&backend, Some(&storage), &changes, 4).unwrap();
assert_eq!(changes_trie_nodes, Some(vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![101] }, vec![1]),
@@ -258,7 +255,7 @@ mod test {
#[test]
fn build_changes_trie_nodes_on_digest_block_l2() {
let (backend, storage, changes) = prepare_for_build();
let changes_trie_nodes = prepare_input::<_, _, _, RlpCodec>(&backend, Some(&storage), &changes, 16).unwrap();
let changes_trie_nodes = prepare_input(&backend, Some(&storage), &changes, 16).unwrap();
assert_eq!(changes_trie_nodes, Some(vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![101] }, vec![1]),
@@ -282,7 +279,7 @@ mod test {
extrinsics: Some(vec![1].into_iter().collect())
});
let changes_trie_nodes = prepare_input::<_, _, _, RlpCodec>(&backend, Some(&storage), &changes, 4).unwrap();
let changes_trie_nodes = prepare_input(&backend, Some(&storage), &changes, 4).unwrap();
assert_eq!(changes_trie_nodes, Some(vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![101] }, vec![1]),
@@ -20,10 +20,9 @@
use std::cell::RefCell;
use std::collections::VecDeque;
use codec::{Decode, Encode};
use hashdb::{HashDB, Hasher};
use hash_db::{HashDB, Hasher};
use heapsize::HeapSizeOf;
use memorydb::MemoryDB;
use patricia_trie::{NodeCodec, Recorder};
use substrate_trie::{Recorder, MemoryDB};
use changes_trie::{Configuration, Storage};
use changes_trie::input::{DigestIndex, ExtrinsicIndex, DigestIndexValue, ExtrinsicIndexValue};
use changes_trie::storage::{TrieBackendAdapter, InMemoryStorage};
@@ -32,7 +31,7 @@ use trie_backend_essence::{TrieBackendEssence};
/// Return changes of given key at given blocks range.
/// `max` is the number of best known block.
pub fn key_changes<S: Storage<H>, H: Hasher, C: NodeCodec<H>>(
pub fn key_changes<S: Storage<H>, H: Hasher>(
config: &Configuration,
storage: &S,
begin: u64,
@@ -52,13 +51,12 @@ pub fn key_changes<S: Storage<H>, H: Hasher, C: NodeCodec<H>>(
_hasher: ::std::marker::PhantomData::<H>::default(),
},
_codec: ::std::marker::PhantomData::<C>::default(),
}.collect()
}
/// Returns proof of changes of given key at given blocks range.
/// `max` is the number of best known block.
pub fn key_changes_proof<S: Storage<H>, H: Hasher, C: NodeCodec<H>>(
pub fn key_changes_proof<S: Storage<H>, H: Hasher>(
config: &Configuration,
storage: &S,
begin: u64,
@@ -79,7 +77,6 @@ pub fn key_changes_proof<S: Storage<H>, H: Hasher, C: NodeCodec<H>>(
_hasher: ::std::marker::PhantomData::<H>::default(),
},
proof_recorder: Default::default(),
_codec: ::std::marker::PhantomData::<C>::default(),
};
// iterate to collect proof
@@ -92,7 +89,7 @@ pub fn key_changes_proof<S: Storage<H>, H: Hasher, C: NodeCodec<H>>(
/// Check key changes proog and return changes of the key at given blocks range.
/// `max` is the number of best known block.
pub fn key_changes_proof_check<S: Storage<H>, H: Hasher, C: NodeCodec<H>>(
pub fn key_changes_proof_check<S: Storage<H>, H: Hasher>(
config: &Configuration,
roots_storage: &S, // TODO: use RootsStorage is only used to gather root
proof: Vec<Vec<u8>>,
@@ -101,7 +98,7 @@ pub fn key_changes_proof_check<S: Storage<H>, H: Hasher, C: NodeCodec<H>>(
max: u64,
key: &[u8]
) -> Result<Vec<(u64, u32)>, String> where H::Out: HeapSizeOf {
let mut proof_db = MemoryDB::<H>::new();
let mut proof_db = MemoryDB::<H>::default(); // TODO: use new for correctness
for item in proof {
proof_db.insert(&item);
}
@@ -119,7 +116,6 @@ pub fn key_changes_proof_check<S: Storage<H>, H: Hasher, C: NodeCodec<H>>(
_hasher: ::std::marker::PhantomData::<H>::default(),
},
_codec: ::std::marker::PhantomData::<C>::default(),
}.collect()
}
@@ -239,28 +235,29 @@ impl<'a, RS: 'a + Storage<H>, S: Storage<H>, H: Hasher> DrilldownIteratorEssence
}
/// Exploring drilldown operator.
struct DrilldownIterator<'a, RS: 'a + Storage<H>, S: 'a + Storage<H>, H: Hasher, C: NodeCodec<H>> {
struct DrilldownIterator<'a, RS: 'a + Storage<H>, S: 'a + Storage<H>, H: Hasher> {
essence: DrilldownIteratorEssence<'a, RS, S, H>,
_codec: ::std::marker::PhantomData<C>,
}
impl<'a, RS: 'a + Storage<H>, S: Storage<H>, H: Hasher, C: NodeCodec<H>> Iterator for DrilldownIterator<'a, RS, S, H, C> where H::Out: HeapSizeOf {
impl<'a, RS: 'a + Storage<H>, S: Storage<H>, H: Hasher> Iterator
for DrilldownIterator<'a, RS, S, H>
where H::Out: HeapSizeOf
{
type Item = Result<(u64, u32), String>;
fn next(&mut self) -> Option<Self::Item> {
self.essence.next(|storage, root, key|
TrieBackendEssence::<_, H, C>::new(TrieBackendAdapter::new(storage), root).storage(key))
TrieBackendEssence::<_, H>::new(TrieBackendAdapter::new(storage), root).storage(key))
}
}
/// Proving drilldown iterator.
struct ProvingDrilldownIterator<'a, RS: 'a + Storage<H>, S: 'a + Storage<H>, H: Hasher, C: NodeCodec<H>> {
struct ProvingDrilldownIterator<'a, RS: 'a + Storage<H>, S: 'a + Storage<H>, H: Hasher> {
essence: DrilldownIteratorEssence<'a, RS, S, H>,
proof_recorder: RefCell<Recorder<H::Out>>,
_codec: ::std::marker::PhantomData<C>,
}
impl<'a, RS: 'a + Storage<H>, S: Storage<H>, H: Hasher, C: NodeCodec<H>> ProvingDrilldownIterator<'a, RS, S, H, C> {
impl<'a, RS: 'a + Storage<H>, S: Storage<H>, H: Hasher> ProvingDrilldownIterator<'a, RS, S, H> {
/// Consume the iterator, extracting the gathered proof in lexicographical order
/// by value.
pub fn extract_proof(self) -> Vec<Vec<u8>> {
@@ -271,14 +268,14 @@ impl<'a, RS: 'a + Storage<H>, S: Storage<H>, H: Hasher, C: NodeCodec<H>> Proving
}
}
impl<'a, RS: 'a + Storage<H>, S: Storage<H>, H: Hasher, C: NodeCodec<H>> Iterator for ProvingDrilldownIterator<'a, RS, S, H, C> where H::Out: HeapSizeOf {
impl<'a, RS: 'a + Storage<H>, S: Storage<H>, H: Hasher> Iterator for ProvingDrilldownIterator<'a, RS, S, H> where H::Out: HeapSizeOf {
type Item = Result<(u64, u32), String>;
fn next(&mut self) -> Option<Self::Item> {
let proof_recorder = &mut *self.proof_recorder.try_borrow_mut()
.expect("only fails when already borrowed; storage() is non-reentrant; qed");
self.essence.next(|storage, root, key|
ProvingBackendEssence::<_, H, C> {
ProvingBackendEssence::<_, H> {
backend: &TrieBackendEssence::new(TrieBackendAdapter::new(storage), root),
proof_recorder,
}.storage(key))
@@ -356,14 +353,14 @@ fn lower_bound_max_digest(
#[cfg(test)]
mod tests {
use primitives::{Blake2Hasher, RlpCodec};
use primitives::Blake2Hasher;
use changes_trie::input::InputPair;
use changes_trie::storage::InMemoryStorage;
use super::*;
fn prepare_for_drilldown() -> (Configuration, InMemoryStorage<Blake2Hasher>) {
let config = Configuration { digest_interval: 4, digest_levels: 2 };
let backend = InMemoryStorage::with_inputs::<RlpCodec>(vec![
let backend = InMemoryStorage::with_inputs(vec![
// digest: 1..4 => [(3, 0)]
(1, vec![]),
(2, vec![]),
@@ -403,7 +400,7 @@ mod tests {
#[test]
fn drilldown_iterator_works() {
let (config, storage) = prepare_for_drilldown();
let drilldown_result = key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher, RlpCodec>(
let drilldown_result = key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher>(
&config, &storage, 0, 100, 1000, &[42]);
assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)]));
@@ -414,16 +411,16 @@ mod tests {
let (config, storage) = prepare_for_drilldown();
storage.clear_storage();
assert!(key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher, RlpCodec>(
assert!(key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher>(
&config, &storage, 0, 100, 1000, &[42]).is_err());
}
#[test]
fn drilldown_iterator_fails_when_range_is_invalid() {
let (config, storage) = prepare_for_drilldown();
assert!(key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher, RlpCodec>(
assert!(key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher>(
&config, &storage, 0, 100, 50, &[42]).is_err());
assert!(key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher, RlpCodec>(
assert!(key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher>(
&config, &storage, 20, 10, 100, &[42]).is_err());
}
@@ -434,7 +431,7 @@ mod tests {
// create drilldown iterator that records all trie nodes during drilldown
let (remote_config, remote_storage) = prepare_for_drilldown();
let remote_proof = key_changes_proof::<InMemoryStorage<Blake2Hasher>, Blake2Hasher, RlpCodec>(
let remote_proof = key_changes_proof::<InMemoryStorage<Blake2Hasher>, Blake2Hasher>(
&remote_config, &remote_storage,
0, 100, 1000, &[42]).unwrap();
@@ -443,7 +440,7 @@ mod tests {
// create drilldown iterator that works the same, but only depends on trie
let (local_config, local_storage) = prepare_for_drilldown();
local_storage.clear_storage();
let local_result = key_changes_proof_check::<InMemoryStorage<Blake2Hasher>, Blake2Hasher, RlpCodec>(
let local_result = key_changes_proof_check::<InMemoryStorage<Blake2Hasher>, Blake2Hasher>(
&local_config, &local_storage, remote_proof,
0, 100, 1000, &[42]);
@@ -41,15 +41,14 @@ mod storage;
pub use self::storage::InMemoryStorage;
pub use self::changes_iterator::{key_changes, key_changes_proof, key_changes_proof_check};
use hashdb::{DBValue, Hasher};
use hash_db::Hasher;
use heapsize::HeapSizeOf;
use patricia_trie::NodeCodec;
use rlp::Encodable;
use backend::Backend;
use primitives;
use changes_trie::build::prepare_input;
use overlayed_changes::OverlayedChanges;
use trie_backend_essence::TrieBackendStorage;
use trie::{DBValue, trie_root};
/// Changes that are made outside of extrinsics are marked with this index;
pub const NO_EXTRINSIC_INDEX: u32 = 0xffffffff;
@@ -68,7 +67,7 @@ pub type Configuration = primitives::ChangesTrieConfiguration;
/// Compute the changes trie root and transaction for given block.
/// Returns None if there's no data to perform computation.
pub fn compute_changes_trie_root<'a, B: Backend<H, C>, S: Storage<H>, H: Hasher, C: NodeCodec<H>>(
pub fn compute_changes_trie_root<'a, B: Backend<H>, S: Storage<H>, H: Hasher>(
backend: &B,
storage: Option<&'a S>,
changes: &OverlayedChanges,
@@ -76,14 +75,14 @@ pub fn compute_changes_trie_root<'a, B: Backend<H, C>, S: Storage<H>, H: Hasher,
) -> Option<(H::Out, Vec<(Vec<u8>, Vec<u8>)>)>
where
&'a S: TrieBackendStorage<H>,
H::Out: Ord + Encodable + HeapSizeOf,
H::Out: Ord + HeapSizeOf,
{
let input_pairs = prepare_input::<B, S, H, C>(backend, storage, changes, block)
let input_pairs = prepare_input::<B, S, H>(backend, storage, changes, block)
.expect("storage is not allowed to fail within runtime")?;
let transaction = input_pairs.into_iter()
.map(Into::into)
.collect::<Vec<_>>();
let root = ::triehash::trie_root::<H, _, _, _>(transaction.iter().map(|(k, v)| (&*k, &*v)));
let root = trie_root::<H, _, _, _>(transaction.iter().map(|(k, v)| (&*k, &*v)));
Some((root, transaction))
}
@@ -17,9 +17,10 @@
//! Changes trie storage utilities.
use std::collections::HashMap;
use hashdb::{Hasher, HashDB, DBValue};
use hash_db::Hasher;
use trie::DBValue;
use heapsize::HeapSizeOf;
use memorydb::MemoryDB;
use trie::MemoryDB;
use parking_lot::RwLock;
use changes_trie::Storage;
use trie_backend_essence::TrieBackendStorage;
@@ -27,8 +28,6 @@ use trie_backend_essence::TrieBackendStorage;
#[cfg(test)]
use backend::insert_into_memory_db;
#[cfg(test)]
use patricia_trie::NodeCodec;
#[cfg(test)]
use changes_trie::input::InputPair;
/// In-memory implementation of changes trie storage.
@@ -64,11 +63,11 @@ impl<H: Hasher> InMemoryStorage<H> where H::Out: HeapSizeOf {
}
#[cfg(test)]
pub fn with_inputs<C: NodeCodec<H>>(inputs: Vec<(u64, Vec<InputPair>)>) -> Self {
pub fn with_inputs(inputs: Vec<(u64, Vec<InputPair>)>) -> Self {
let mut mdb = MemoryDB::default();
let mut roots = HashMap::new();
for (block, pairs) in inputs {
let root = insert_into_memory_db::<H, C, _>(&mut mdb, pairs.into_iter().map(Into::into));
let root = insert_into_memory_db::<H, _>(&mut mdb, pairs.into_iter().map(Into::into));
if let Some(root) = root {
roots.insert(block, root);
}
@@ -84,7 +83,7 @@ impl<H: Hasher> InMemoryStorage<H> where H::Out: HeapSizeOf {
#[cfg(test)]
pub fn clear_storage(&self) {
self.data.write().mdb = MemoryDB::new();
self.data.write().mdb = MemoryDB::default(); // use new to be more correct
}
/// Insert changes trie for given block.
@@ -101,7 +100,7 @@ impl<H: Hasher> Storage<H> for InMemoryStorage<H> where H::Out: HeapSizeOf {
}
fn get(&self, key: &H::Out) -> Result<Option<DBValue>, String> {
Ok(HashDB::<H>::get(&self.data.read().mdb, key))
MemoryDB::<H>::get(&self.data.read().mdb, key)
}
}
+22 -26
View File
@@ -20,10 +20,8 @@ use std::{error, fmt, cmp::Ord};
use backend::Backend;
use changes_trie::{Storage as ChangesTrieStorage, compute_changes_trie_root};
use {Externalities, OverlayedChanges};
use hashdb::Hasher;
use memorydb::MemoryDB;
use rlp::Encodable;
use patricia_trie::{NodeCodec, TrieDBMut, TrieMut};
use hash_db::Hasher;
use substrate_trie::{MemoryDB, TrieDBMut, TrieMut};
use heapsize::HeapSizeOf;
const EXT_NOT_ALLOWED_TO_FAIL: &'static str = "Externalities not allowed to fail within runtime";
@@ -58,11 +56,11 @@ impl<B: error::Error, E: error::Error> error::Error for Error<B, E> {
}
/// Wraps a read-only backend, call executor, and current overlayed changes.
pub struct Ext<'a, H, C, B, T>
pub struct Ext<'a, H, B, T>
where
H: Hasher,
C: NodeCodec<H>,
B: 'a + Backend<H, C>,
B: 'a + Backend<H>,
T: 'a + ChangesTrieStorage<H>,
{
/// The overlayed changes to write to.
@@ -83,13 +81,12 @@ where
changes_trie_transaction: Option<(u64, MemoryDB<H>, H::Out)>,
}
impl<'a, H, C, B, T> Ext<'a, H, C, B, T>
impl<'a, H, B, T> Ext<'a, H, B, T>
where
H: Hasher,
C: NodeCodec<H>,
B: 'a + Backend<H, C>,
B: 'a + Backend<H>,
T: 'a + ChangesTrieStorage<H>,
H::Out: Ord + Encodable + HeapSizeOf,
H::Out: Ord + HeapSizeOf,
{
/// Create a new `Ext` from overlayed changes and read-only backend
pub fn new(overlay: &'a mut OverlayedChanges, backend: &'a B, changes_trie_storage: Option<&'a T>) -> Self {
@@ -128,11 +125,11 @@ where
}
#[cfg(test)]
impl<'a, H, C, B, T> Ext<'a, H, C, B, T>
impl<'a, H, B, T> Ext<'a, H, B, T>
where
H: Hasher,
C: NodeCodec<H>,
B: 'a + Backend<H,C>,
B: 'a + Backend<H>,
T: 'a + ChangesTrieStorage<H>,
{
pub fn storage_pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
@@ -149,13 +146,12 @@ where
}
}
impl<'a, B: 'a, T: 'a, H, C> Externalities<H> for Ext<'a, H, C, B, T>
impl<'a, B: 'a, T: 'a, H> Externalities<H> for Ext<'a, H, B, T>
where
H: Hasher,
C: NodeCodec<H>,
B: 'a + Backend<H, C>,
B: 'a + Backend<H>,
T: 'a + ChangesTrieStorage<H>,
H::Out: Ord + Encodable + HeapSizeOf,
H::Out: Ord + HeapSizeOf,
{
fn storage(&self, key: &[u8]) -> Option<Vec<u8>> {
self.overlay.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(||
@@ -201,7 +197,7 @@ where
}
fn storage_changes_root(&mut self, block: u64) -> Option<H::Out> {
let root_and_tx = compute_changes_trie_root::<_, T, H, C>(
let root_and_tx = compute_changes_trie_root::<_, T, H>(
self.backend,
self.changes_trie_storage.clone(),
self.overlay,
@@ -209,9 +205,9 @@ where
);
let root_and_tx = root_and_tx.map(|(root, changes)| {
let mut calculated_root = Default::default();
let mut mdb = MemoryDB::new();
let mut mdb = MemoryDB::default(); // TODO: use new for correctness
{
let mut trie = TrieDBMut::<H, C>::new(&mut mdb, &mut calculated_root);
let mut trie = TrieDBMut::<H>::new(&mut mdb, &mut calculated_root);
for (key, value) in changes {
trie.insert(&key, &value).expect(EXT_NOT_ALLOWED_TO_FAIL);
}
@@ -228,7 +224,7 @@ where
#[cfg(test)]
mod tests {
use codec::Encode;
use primitives::{Blake2Hasher, RlpCodec};
use primitives::{Blake2Hasher};
use primitives::storage::well_known_keys::EXTRINSIC_INDEX;
use backend::InMemory;
use changes_trie::{Configuration as ChangesTrieConfiguration,
@@ -236,9 +232,9 @@ mod tests {
use overlayed_changes::OverlayedValue;
use super::*;
type TestBackend = InMemory<Blake2Hasher, RlpCodec>;
type TestBackend = InMemory<Blake2Hasher>;
type TestChangesTrieStorage = InMemoryChangesTrieStorage<Blake2Hasher>;
type TestExt<'a> = Ext<'a, Blake2Hasher, RlpCodec, TestBackend, TestChangesTrieStorage>;
type TestExt<'a> = Ext<'a, Blake2Hasher, TestBackend, TestChangesTrieStorage>;
fn prepare_overlay_with_changes() -> OverlayedChanges {
OverlayedChanges {
@@ -285,7 +281,7 @@ mod tests {
let backend = TestBackend::default();
let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage));
assert_eq!(ext.storage_changes_root(100),
Some(hex!("62d1aacd26ef215f0d846f96d468cfcf445a20ed792cb43b3ed2e285fac7aced").into()));
Some(hex!("5b829920b9c8d554a19ee2a1ba593c4f2ee6fc32822d083e04236d693e8358d5").into()));
}
#[test]
@@ -296,6 +292,6 @@ mod tests {
let backend = TestBackend::default();
let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage));
assert_eq!(ext.storage_changes_root(100),
Some(hex!("16776812bae4843437f7ec9f054f78ea8305e966da41b718cd6b847e95480d5c").into()));
Some(hex!("bcf494e41e29a15c9ae5caa053fe3cb8b446ee3e02a254efbdec7a19235b76e4").into()));
}
}
+42 -48
View File
@@ -26,10 +26,9 @@ extern crate hex_literal;
#[macro_use]
extern crate log;
extern crate hashdb;
extern crate memorydb;
extern crate triehash;
extern crate patricia_trie;
extern crate hash_db;
extern crate memory_db;
extern crate substrate_trie;
extern crate byteorder;
extern crate parking_lot;
@@ -37,11 +36,10 @@ extern crate rlp;
extern crate heapsize;
extern crate substrate_primitives as primitives;
extern crate parity_codec as codec;
extern crate substrate_trie as trie;
use std::fmt;
use hashdb::Hasher;
use patricia_trie::NodeCodec;
use rlp::Encodable;
use hash_db::Hasher;
use heapsize::HeapSizeOf;
use codec::Decode;
use primitives::storage::well_known_keys;
@@ -55,7 +53,7 @@ mod proving_backend;
mod trie_backend;
mod trie_backend_essence;
pub use patricia_trie::{TrieMut, TrieDBMut};
pub use trie::{TrieMut, TrieDBMut, DBValue, MemoryDB};
pub use testing::TestExternalities;
pub use ext::Ext;
pub use backend::Backend;
@@ -64,7 +62,7 @@ pub use changes_trie::{Storage as ChangesTrieStorage,
key_changes, key_changes_proof, key_changes_proof_check};
pub use overlayed_changes::OverlayedChanges;
pub use trie_backend_essence::Storage;
pub use trie_backend::{TrieBackend, DBValue};
pub use trie_backend::TrieBackend;
/// State Machine Error bound.
///
@@ -124,10 +122,10 @@ pub trait Externalities<H: Hasher> {
fn chain_id(&self) -> u64;
/// Get the trie root of the current storage map.
fn storage_root(&mut self) -> H::Out where H::Out: Ord + Encodable;
fn storage_root(&mut self) -> H::Out where H::Out: Ord;
/// Get the change trie root of the current storage overlay at given block.
fn storage_changes_root(&mut self, block: u64) -> Option<H::Out> where H::Out: Ord + Encodable;
fn storage_changes_root(&mut self, block: u64) -> Option<H::Out> where H::Out: Ord;
}
/// Code execution engine.
@@ -197,7 +195,7 @@ pub fn always_wasm<E>() -> ExecutionManager<fn(Result<Vec<u8>, E>, Result<Vec<u8
///
/// Note: changes to code will be in place if this call is made again. For running partial
/// blocks (e.g. a transaction at a time), ensure a different method is used.
pub fn execute<H, C, B, T, Exec>(
pub fn execute<H, B, T, Exec>(
backend: &B,
changes_trie_storage: Option<&T>,
overlay: &mut OverlayedChanges,
@@ -205,14 +203,13 @@ pub fn execute<H, C, B, T, Exec>(
method: &str,
call_data: &[u8],
strategy: ExecutionStrategy,
) -> Result<(Vec<u8>, B::Transaction, Option<memorydb::MemoryDB<H>>), Box<Error>>
) -> Result<(Vec<u8>, B::Transaction, Option<MemoryDB<H>>), Box<Error>>
where
H: Hasher,
C: NodeCodec<H>,
Exec: CodeExecutor<H>,
B: Backend<H, C>,
B: Backend<H>,
T: ChangesTrieStorage<H>,
H::Out: Ord + Encodable + HeapSizeOf,
H::Out: Ord + HeapSizeOf,
{
execute_using_consensus_failure_handler(
backend,
@@ -240,7 +237,7 @@ where
///
/// Note: changes to code will be in place if this call is made again. For running partial
/// blocks (e.g. a transaction at a time), ensure a different method is used.
pub fn execute_using_consensus_failure_handler<H, C, B, T, Exec, Handler>(
pub fn execute_using_consensus_failure_handler<H, B, T, Exec, Handler>(
backend: &B,
changes_trie_storage: Option<&T>,
overlay: &mut OverlayedChanges,
@@ -248,14 +245,13 @@ pub fn execute_using_consensus_failure_handler<H, C, B, T, Exec, Handler>(
method: &str,
call_data: &[u8],
manager: ExecutionManager<Handler>,
) -> Result<(Vec<u8>, B::Transaction, Option<memorydb::MemoryDB<H>>), Box<Error>>
) -> Result<(Vec<u8>, B::Transaction, Option<MemoryDB<H>>), Box<Error>>
where
H: Hasher,
C: NodeCodec<H>,
Exec: CodeExecutor<H>,
B: Backend<H, C>,
B: Backend<H>,
T: ChangesTrieStorage<H>,
H::Out: Ord + Encodable + HeapSizeOf,
H::Out: Ord + HeapSizeOf,
Handler: FnOnce(Result<Vec<u8>, Exec::Error>, Result<Vec<u8>, Exec::Error>) -> Result<Vec<u8>, Exec::Error>
{
let strategy: ExecutionStrategy = (&manager).into();
@@ -352,7 +348,7 @@ where
///
/// Note: changes to code will be in place if this call is made again. For running partial
/// blocks (e.g. a transaction at a time), ensure a different method is used.
pub fn prove_execution<B, H, C, Exec>(
pub fn prove_execution<B, H, Exec>(
backend: B,
overlay: &mut OverlayedChanges,
exec: &Exec,
@@ -360,16 +356,15 @@ pub fn prove_execution<B, H, C, Exec>(
call_data: &[u8],
) -> Result<(Vec<u8>, Vec<Vec<u8>>), Box<Error>>
where
B: Backend<H, C>,
B: Backend<H>,
H: Hasher,
Exec: CodeExecutor<H>,
C: NodeCodec<H>,
H::Out: Ord + Encodable + HeapSizeOf,
H::Out: Ord + HeapSizeOf,
{
let trie_backend = backend.try_into_trie_backend()
.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<Error>)?;
let proving_backend = proving_backend::ProvingBackend::new(trie_backend);
let (result, _, _) = execute::<H, C, _, changes_trie::InMemoryStorage<H>, _>(
let (result, _, _) = execute::<H, _, changes_trie::InMemoryStorage<H>, _>(
&proving_backend,
None,
overlay,
@@ -383,7 +378,7 @@ where
}
/// Check execution proof, generated by `prove_execution` call.
pub fn execution_proof_check<H, C, Exec>(
pub fn execution_proof_check<H, Exec>(
root: H::Out,
proof: Vec<Vec<u8>>,
overlay: &mut OverlayedChanges,
@@ -393,45 +388,44 @@ pub fn execution_proof_check<H, C, Exec>(
) -> Result<Vec<u8>, Box<Error>>
where
H: Hasher,
C: NodeCodec<H>,
Exec: CodeExecutor<H>,
H::Out: Ord + Encodable + HeapSizeOf,
H::Out: Ord + HeapSizeOf,
{
let backend = proving_backend::create_proof_check_backend::<H, C>(root.into(), proof)?;
execute::<H, C, _, changes_trie::InMemoryStorage<H>, _>(&backend, None, overlay, exec, method, call_data, ExecutionStrategy::NativeWhenPossible)
let backend = proving_backend::create_proof_check_backend::<H>(root.into(), proof)?;
execute::<H, _, changes_trie::InMemoryStorage<H>, _>(&backend, None, overlay, exec, method, call_data, ExecutionStrategy::NativeWhenPossible)
.map(|(result, _, _)| result)
}
/// Generate storage read proof.
pub fn prove_read<B, H, C>(
pub fn prove_read<B, H>(
backend: B,
key: &[u8]
) -> Result<(Option<Vec<u8>>, Vec<Vec<u8>>), Box<Error>>
where
B: Backend<H, C>,
B: Backend<H>,
H: Hasher,
C: NodeCodec<H>,
H::Out: Ord + Encodable + HeapSizeOf
H::Out: Ord + HeapSizeOf
{
let trie_backend = backend.try_into_trie_backend()
.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<Error>)?;
let proving_backend = proving_backend::ProvingBackend::<_, H, C>::new(trie_backend);
let proving_backend = proving_backend::ProvingBackend::<_, H>::new(trie_backend);
let result = proving_backend.storage(key).map_err(|e| Box::new(e) as Box<Error>)?;
Ok((result, proving_backend.extract_proof()))
}
/// Check storage read proof, generated by `prove_read` call.
pub fn read_proof_check<H, C>(
pub fn read_proof_check<H>(
root: H::Out,
proof: Vec<Vec<u8>>,
key: &[u8],
) -> Result<Option<Vec<u8>>, Box<Error>>
where
H: Hasher,
C: NodeCodec<H>,
H::Out: Ord + Encodable + HeapSizeOf
H::Out: Ord + HeapSizeOf
{
let backend = proving_backend::create_proof_check_backend::<H, C>(root, proof)?;
let backend = proving_backend::create_proof_check_backend::<H>(root, proof)?;
backend.storage(key).map_err(|e| Box::new(e) as Box<Error>)
}
@@ -453,12 +447,12 @@ pub(crate) fn set_changes_trie_config(overlay: &mut OverlayedChanges, config: Op
}
/// Reads storage value from overlay or from the backend.
fn try_read_overlay_value<H, C, B>(overlay: &OverlayedChanges, backend: &B, key: &[u8])
fn try_read_overlay_value<H, B>(overlay: &OverlayedChanges, backend: &B, key: &[u8])
-> Result<Option<Vec<u8>>, Box<Error>>
where
H: Hasher,
C: NodeCodec<H>,
B: Backend<H, C>,
B: Backend<H>,
{
match overlay.storage(key).map(|x| x.map(|x| x.to_vec())) {
Some(value) => Ok(value),
@@ -474,7 +468,7 @@ mod tests {
use super::backend::InMemory;
use super::ext::Ext;
use super::changes_trie::InMemoryStorage as InMemoryChangesTrieStorage;
use primitives::{Blake2Hasher, RlpCodec};
use primitives::{Blake2Hasher};
struct DummyCodeExecutor {
native_available: bool,
@@ -566,7 +560,7 @@ mod tests {
&mut Default::default(), &executor, "test", &[]).unwrap();
// check proof locally
let local_result = execution_proof_check::<Blake2Hasher, RlpCodec, _>(remote_root, remote_proof,
let local_result = execution_proof_check::<Blake2Hasher, _>(remote_root, remote_proof,
&mut Default::default(), &executor, "test", &[]).unwrap();
// check that both results are correct
@@ -582,7 +576,7 @@ mod tests {
b"abc".to_vec() => b"2".to_vec(),
b"bbb".to_vec() => b"3".to_vec()
];
let backend = InMemory::<Blake2Hasher, RlpCodec>::from(initial).try_into_trie_backend().unwrap();
let backend = InMemory::<Blake2Hasher>::from(initial).try_into_trie_backend().unwrap();
let mut overlay = OverlayedChanges {
committed: map![
b"aba".to_vec() => Some(b"1312".to_vec()).into(),
@@ -623,8 +617,8 @@ mod tests {
let remote_root = remote_backend.storage_root(::std::iter::empty()).0;
let remote_proof = prove_read(remote_backend, b"value2").unwrap().1;
// check proof locally
let local_result1 = read_proof_check::<Blake2Hasher, RlpCodec>(remote_root, remote_proof.clone(), b"value2").unwrap();
let local_result2 = read_proof_check::<Blake2Hasher, RlpCodec>(remote_root, remote_proof.clone(), &[0xff]).is_ok();
let local_result1 = read_proof_check::<Blake2Hasher>(remote_root, remote_proof.clone(), b"value2").unwrap();
let local_result2 = read_proof_check::<Blake2Hasher>(remote_root, remote_proof.clone(), &[0xff]).is_ok();
// check that results are correct
assert_eq!(local_result1, Some(vec![24]));
assert_eq!(local_result2, false);
@@ -201,7 +201,7 @@ impl From<Option<Vec<u8>>> for OverlayedValue {
#[cfg(test)]
mod tests {
use primitives::{Blake2Hasher, RlpCodec, H256};
use primitives::{Blake2Hasher, H256};
use primitives::storage::well_known_keys::EXTRINSIC_INDEX;
use backend::InMemory;
use changes_trie::InMemoryStorage as InMemoryChangesTrieStorage;
@@ -251,7 +251,7 @@ mod tests {
(b"dogglesworth".to_vec(), b"catXXX".to_vec()),
(b"doug".to_vec(), b"notadog".to_vec()),
].into_iter().collect();
let backend = InMemory::<Blake2Hasher, RlpCodec>::from(initial);
let backend = InMemory::<Blake2Hasher>::from(initial);
let mut overlay = OverlayedChanges {
committed: vec![
(b"dog".to_vec(), Some(b"puppy".to_vec()).into()),
@@ -267,7 +267,7 @@ mod tests {
let changes_trie_storage = InMemoryChangesTrieStorage::new();
let mut ext = Ext::new(&mut overlay, &backend, Some(&changes_trie_storage));
const ROOT: [u8; 32] = hex!("6ca394ff9b13d6690a51dea30b1b5c43108e52944d30b9095227c49bae03ff8b");
const ROOT: [u8; 32] = hex!("0b41e488cccbd67d1f1089592c2c235f5c5399b053f7fe9152dd4b5f279914cd");
assert_eq!(ext.storage_root(), H256(ROOT));
}
@@ -17,28 +17,27 @@
//! Proving state machine backend.
use std::cell::RefCell;
use hashdb::{Hasher, HashDB};
use hash_db::Hasher;
use heapsize::HeapSizeOf;
use memorydb::MemoryDB;
use patricia_trie::{TrieDB, Trie, Recorder, NodeCodec};
use rlp::Encodable;
use hash_db::HashDB;
use trie::{TrieDB, Trie, Recorder, MemoryDB};
use trie_backend::TrieBackend;
use trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage};
use {Error, ExecutionError, Backend};
/// Patricia trie-based backend essence which also tracks all touched storage trie values.
/// These can be sent to remote node and used as a proof of execution.
pub struct ProvingBackendEssence<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher, C: 'a + NodeCodec<H>> {
pub(crate) backend: &'a TrieBackendEssence<S, H, C>,
pub struct ProvingBackendEssence<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> {
pub(crate) backend: &'a TrieBackendEssence<S, H>,
pub(crate) proof_recorder: &'a mut Recorder<H::Out>,
}
impl<'a, S, H, C> ProvingBackendEssence<'a, S, H, C>
impl<'a, S, H> ProvingBackendEssence<'a, S, H>
where
S: TrieBackendStorage<H>,
H: Hasher,
H::Out: HeapSizeOf,
C: NodeCodec<H>,
{
pub fn storage(&mut self, key: &[u8]) -> Result<Option<Vec<u8>>, String> {
let mut read_overlay = MemoryDB::default();
@@ -49,7 +48,7 @@ impl<'a, S, H, C> ProvingBackendEssence<'a, S, H, C>
let map_e = |e| format!("Trie lookup error: {}", e);
TrieDB::<H, C>::new(&eph, self.backend.root()).map_err(map_e)?
TrieDB::<H>::new(&eph, self.backend.root()).map_err(map_e)?
.get_with(key, &mut *self.proof_recorder)
.map(|x| x.map(|val| val.to_vec()))
.map_err(map_e)
@@ -58,14 +57,14 @@ impl<'a, S, H, C> ProvingBackendEssence<'a, S, H, C>
/// Patricia trie-based backend which also tracks all touched storage trie values.
/// These can be sent to remote node and used as a proof of execution.
pub struct ProvingBackend<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> {
backend: TrieBackend<S, H, C>,
pub struct ProvingBackend<S: TrieBackendStorage<H>, H: Hasher> {
backend: TrieBackend<S, H>,
proof_recorder: RefCell<Recorder<H::Out>>,
}
impl<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> ProvingBackend<S, H, C> {
impl<S: TrieBackendStorage<H>, H: Hasher> ProvingBackend<S, H> {
/// Create new proving backend.
pub fn new(backend: TrieBackend<S, H, C>) -> Self {
pub fn new(backend: TrieBackend<S, H>) -> Self {
ProvingBackend {
backend,
proof_recorder: RefCell::new(Recorder::new()),
@@ -82,12 +81,11 @@ impl<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> ProvingBackend<S, H,
}
}
impl<S, H, C> Backend<H, C> for ProvingBackend<S, H, C>
impl<S, H> Backend<H> for ProvingBackend<S, H>
where
S: TrieBackendStorage<H>,
H: Hasher,
C: NodeCodec<H>,
H::Out: Ord + Encodable + HeapSizeOf,
H::Out: Ord + HeapSizeOf,
{
type Error = String;
type Transaction = MemoryDB<H>;
@@ -115,22 +113,21 @@ impl<S, H, C> Backend<H, C> for ProvingBackend<S, H, C>
self.backend.storage_root(delta)
}
fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H, C>> {
fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H>> {
None
}
}
/// Create proof check backend.
pub fn create_proof_check_backend<H, C>(
pub fn create_proof_check_backend<H>(
root: H::Out,
proof: Vec<Vec<u8>>
) -> Result<TrieBackend<MemoryDB<H>, H, C>, Box<Error>>
) -> Result<TrieBackend<MemoryDB<H>, H>, Box<Error>>
where
H: Hasher,
C: NodeCodec<H>,
H::Out: HeapSizeOf,
{
let mut db = MemoryDB::new();
let mut db = MemoryDB::default(); // TODO: use new for correctness
for item in proof {
db.insert(&item);
}
@@ -147,9 +144,9 @@ mod tests {
use backend::{InMemory};
use trie_backend::tests::test_trie;
use super::*;
use primitives::{Blake2Hasher, RlpCodec};
use primitives::{Blake2Hasher};
fn test_proving() -> ProvingBackend<MemoryDB<Blake2Hasher>, Blake2Hasher, RlpCodec> {
fn test_proving() -> ProvingBackend<MemoryDB<Blake2Hasher>, Blake2Hasher> {
ProvingBackend::new(test_trie())
}
@@ -167,7 +164,7 @@ mod tests {
#[test]
fn proof_is_invalid_when_does_not_contains_root() {
assert!(create_proof_check_backend::<Blake2Hasher, RlpCodec>(1.into(), vec![]).is_err());
assert!(create_proof_check_backend::<Blake2Hasher>(1.into(), vec![]).is_err());
}
#[test]
@@ -186,7 +183,7 @@ mod tests {
#[test]
fn proof_recorded_and_checked() {
let contents = (0..64).map(|i| (vec![i], Some(vec![i]))).collect::<Vec<_>>();
let in_memory = InMemory::<Blake2Hasher, RlpCodec>::default();
let in_memory = InMemory::<Blake2Hasher>::default();
let in_memory = in_memory.update(contents);
let in_memory_root = in_memory.storage_root(::std::iter::empty()).0;
(0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i]));
@@ -201,7 +198,7 @@ mod tests {
let proof = proving.extract_proof();
let proof_check = create_proof_check_backend::<Blake2Hasher, RlpCodec>(in_memory_root.into(), proof).unwrap();
let proof_check = create_proof_check_backend::<Blake2Hasher>(in_memory_root.into(), proof).unwrap();
assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42]);
}
}
+17 -22
View File
@@ -18,25 +18,22 @@
use std::collections::HashMap;
use std::iter::FromIterator;
use hashdb::Hasher;
use hash_db::Hasher;
use heapsize::HeapSizeOf;
use patricia_trie::NodeCodec;
use rlp::Encodable;
use triehash::trie_root;
use trie::trie_root;
use backend::InMemory;
use changes_trie::{compute_changes_trie_root, InMemoryStorage as ChangesTrieInMemoryStorage};
use primitives::storage::well_known_keys::CHANGES_TRIE_CONFIG;
use super::{Externalities, OverlayedChanges};
/// Simple HashMap-based Externalities impl.
pub struct TestExternalities<H: Hasher, C: NodeCodec<H>> where H::Out: HeapSizeOf {
pub struct TestExternalities<H: Hasher> where H::Out: HeapSizeOf {
inner: HashMap<Vec<u8>, Vec<u8>>,
changes_trie_storage: ChangesTrieInMemoryStorage<H>,
changes: OverlayedChanges,
_codec: ::std::marker::PhantomData<C>,
}
impl<H: Hasher, C: NodeCodec<H>> TestExternalities<H, C> where H::Out: HeapSizeOf {
impl<H: Hasher> TestExternalities<H> where H::Out: HeapSizeOf {
/// Create a new instance of `TestExternalities`
pub fn new(inner: HashMap<Vec<u8>, Vec<u8>>) -> Self {
let mut overlay = OverlayedChanges::default();
@@ -49,7 +46,6 @@ impl<H: Hasher, C: NodeCodec<H>> TestExternalities<H, C> where H::Out: HeapSizeO
inner,
changes_trie_storage: ChangesTrieInMemoryStorage::new(),
changes: overlay,
_codec: Default::default(),
}
}
@@ -59,19 +55,19 @@ impl<H: Hasher, C: NodeCodec<H>> TestExternalities<H, C> where H::Out: HeapSizeO
}
}
impl<H: Hasher, C: NodeCodec<H>> ::std::fmt::Debug for TestExternalities<H, C> where H::Out: HeapSizeOf {
impl<H: Hasher> ::std::fmt::Debug for TestExternalities<H> where H::Out: HeapSizeOf {
fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
write!(f, "{:?}", self.inner)
}
}
impl<H: Hasher, C: NodeCodec<H>> PartialEq for TestExternalities<H, C> where H::Out: HeapSizeOf {
fn eq(&self, other: &TestExternalities<H, C>) -> bool {
impl<H: Hasher> PartialEq for TestExternalities<H> where H::Out: HeapSizeOf {
fn eq(&self, other: &TestExternalities<H>) -> bool {
self.inner.eq(&other.inner)
}
}
impl<H: Hasher, C: NodeCodec<H>> FromIterator<(Vec<u8>, Vec<u8>)> for TestExternalities<H, C> where H::Out: HeapSizeOf {
impl<H: Hasher> FromIterator<(Vec<u8>, Vec<u8>)> for TestExternalities<H> where H::Out: HeapSizeOf {
fn from_iter<I: IntoIterator<Item=(Vec<u8>, Vec<u8>)>>(iter: I) -> Self {
let mut t = Self::new(Default::default());
for i in iter {
@@ -81,28 +77,27 @@ impl<H: Hasher, C: NodeCodec<H>> FromIterator<(Vec<u8>, Vec<u8>)> for TestExtern
}
}
impl<H: Hasher, C: NodeCodec<H>> Default for TestExternalities<H, C> where H::Out: HeapSizeOf {
impl<H: Hasher> Default for TestExternalities<H> where H::Out: HeapSizeOf {
fn default() -> Self { Self::new(Default::default()) }
}
impl<H: Hasher, C: NodeCodec<H>> From<TestExternalities<H, C>> for HashMap<Vec<u8>, Vec<u8>> where H::Out: HeapSizeOf {
fn from(tex: TestExternalities<H, C>) -> Self {
impl<H: Hasher> From<TestExternalities<H>> for HashMap<Vec<u8>, Vec<u8>> where H::Out: HeapSizeOf {
fn from(tex: TestExternalities<H>) -> Self {
tex.inner.into()
}
}
impl<H: Hasher, C: NodeCodec<H>> From< HashMap<Vec<u8>, Vec<u8>> > for TestExternalities<H, C> where H::Out: HeapSizeOf {
impl<H: Hasher> From< HashMap<Vec<u8>, Vec<u8>> > for TestExternalities<H> where H::Out: HeapSizeOf {
fn from(hashmap: HashMap<Vec<u8>, Vec<u8>>) -> Self {
TestExternalities {
inner: hashmap,
changes_trie_storage: ChangesTrieInMemoryStorage::new(),
changes: Default::default(),
_codec: ::std::marker::PhantomData::<C>::default(),
}
}
}
impl<H: Hasher, C: NodeCodec<H>> Externalities<H> for TestExternalities<H, C> where H::Out: Ord + Encodable + HeapSizeOf {
impl<H: Hasher> Externalities<H> for TestExternalities<H> where H::Out: Ord + HeapSizeOf {
fn storage(&self, key: &[u8]) -> Option<Vec<u8>> {
self.inner.get(key).map(|x| x.to_vec())
}
@@ -127,7 +122,7 @@ impl<H: Hasher, C: NodeCodec<H>> Externalities<H> for TestExternalities<H, C> wh
}
fn storage_changes_root(&mut self, block: u64) -> Option<H::Out> {
compute_changes_trie_root::<_, _, H, C>(
compute_changes_trie_root::<_, _, H>(
&InMemory::default(),
Some(&self.changes_trie_storage),
&self.changes,
@@ -139,15 +134,15 @@ impl<H: Hasher, C: NodeCodec<H>> Externalities<H> for TestExternalities<H, C> wh
#[cfg(test)]
mod tests {
use super::*;
use primitives::{Blake2Hasher, RlpCodec, H256};
use primitives::{Blake2Hasher, H256};
#[test]
fn commit_should_work() {
let mut ext = TestExternalities::<Blake2Hasher, RlpCodec>::default();
let mut ext = TestExternalities::<Blake2Hasher>::default();
ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec());
ext.set_storage(b"dog".to_vec(), b"puppy".to_vec());
ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec());
const ROOT: [u8; 32] = hex!("6ca394ff9b13d6690a51dea30b1b5c43108e52944d30b9095227c49bae03ff8b");
const ROOT: [u8; 32] = hex!("0b41e488cccbd67d1f1089592c2c235f5c5399b053f7fe9152dd4b5f279914cd");
assert_eq!(ext.storage_root(), H256(ROOT));
}
}
@@ -16,22 +16,18 @@
//! Trie-based state machine backend.
use hashdb::Hasher;
use hash_db::Hasher;
use heapsize::HeapSizeOf;
use memorydb::MemoryDB;
use rlp::Encodable;
use patricia_trie::{TrieDB, TrieDBMut, TrieError, Trie, TrieMut, NodeCodec};
use trie::{TrieDB, TrieDBMut, TrieError, Trie, TrieMut, MemoryDB};
use trie_backend_essence::{TrieBackendEssence, TrieBackendStorage, Ephemeral};
use {Backend};
pub use hashdb::DBValue;
/// Patricia trie-based backend. Transaction type is an overlay of changes to commit.
pub struct TrieBackend<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> {
essence: TrieBackendEssence<S, H, C>,
pub struct TrieBackend<S: TrieBackendStorage<H>, H: Hasher> {
essence: TrieBackendEssence<S, H>,
}
impl<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> TrieBackend<S, H, C> where H::Out: HeapSizeOf {
impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackend<S, H> where H::Out: HeapSizeOf {
/// Create new trie-based backend.
pub fn new(storage: S, root: H::Out) -> Self {
TrieBackend {
@@ -40,7 +36,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> TrieBackend<S, H, C>
}
/// Get backend essence reference.
pub fn essence(&self) -> &TrieBackendEssence<S, H, C> {
pub fn essence(&self) -> &TrieBackendEssence<S, H> {
&self.essence
}
@@ -57,9 +53,8 @@ impl<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> TrieBackend<S, H, C>
impl super::Error for String {}
impl<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> Backend<H, C> for TrieBackend<S, H, C>
where
H::Out: Ord + Encodable + HeapSizeOf,
impl<S: TrieBackendStorage<H>, H: Hasher> Backend<H> for TrieBackend<S, H> where
H::Out: Ord + HeapSizeOf,
{
type Error = String;
type Transaction = MemoryDB<H>;
@@ -74,11 +69,11 @@ impl<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> Backend<H, C> for Tri
}
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
let mut read_overlay = MemoryDB::new();
let mut read_overlay = MemoryDB::default(); // TODO: use new for correctness
let eph = Ephemeral::new(self.essence.backend_storage(), &mut read_overlay);
let collect_all = || -> Result<_, Box<TrieError<H::Out, C::Error>>> {
let trie = TrieDB::<H, C>::new(&eph, self.essence.root())?;
let collect_all = || -> Result<_, Box<TrieError<H::Out>>> {
let trie = TrieDB::<H>::new(&eph, self.essence.root())?;
let mut v = Vec::new();
for x in trie.iter()? {
let (key, value) = x?;
@@ -108,7 +103,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> Backend<H, C> for Tri
&mut write_overlay,
);
let mut trie = TrieDBMut::<H, C>::from_existing(&mut eph, &mut root).expect("prior state root to exist"); // TODO: handle gracefully
let mut trie = TrieDBMut::<H>::from_existing(&mut eph, &mut root).expect("prior state root to exist"); // TODO: handle gracefully
for (key, change) in delta {
let result = match change {
Some(val) => trie.insert(&key, &val),
@@ -124,7 +119,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> Backend<H, C> for Tri
(root, write_overlay)
}
fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H, C>> {
fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H>> {
Some(self)
}
}
@@ -132,14 +127,14 @@ impl<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> Backend<H, C> for Tri
#[cfg(test)]
pub mod tests {
use std::collections::HashSet;
use primitives::{Blake2Hasher, RlpCodec, H256};
use primitives::{Blake2Hasher, H256};
use super::*;
fn test_db() -> (MemoryDB<Blake2Hasher>, H256) {
let mut root = H256::default();
let mut mdb = MemoryDB::<Blake2Hasher>::new();
let mut mdb = MemoryDB::<Blake2Hasher>::default(); // TODO: use new() to be more correct
{
let mut trie = TrieDBMut::<_, RlpCodec>::new(&mut mdb, &mut root);
let mut trie = TrieDBMut::new(&mut mdb, &mut root);
trie.insert(b"key", b"value").expect("insert failed");
trie.insert(b"value1", &[42]).expect("insert failed");
trie.insert(b"value2", &[24]).expect("insert failed");
@@ -151,7 +146,7 @@ pub mod tests {
(mdb, root)
}
pub(crate) fn test_trie() -> TrieBackend<MemoryDB<Blake2Hasher>, Blake2Hasher, RlpCodec> {
pub(crate) fn test_trie() -> TrieBackend<MemoryDB<Blake2Hasher>, Blake2Hasher> {
let (mdb, root) = test_db();
TrieBackend::new(mdb, root)
}
@@ -173,8 +168,8 @@ pub mod tests {
#[test]
fn pairs_are_empty_on_empty_storage() {
assert!(TrieBackend::<MemoryDB<Blake2Hasher>, Blake2Hasher, RlpCodec>::new(
MemoryDB::new(),
assert!(TrieBackend::<MemoryDB<Blake2Hasher>, Blake2Hasher>::new(
MemoryDB::default(), // TODO: use new() to be more correct
Default::default(),
).pairs().is_empty());
}
@@ -18,13 +18,11 @@
//! from storage.
use std::collections::HashMap;
use std::marker::PhantomData;
use std::ops::Deref;
use std::sync::Arc;
use hashdb::{Hasher, DBValue, AsHashDB, HashDB};
use hash_db::{self, Hasher};
use heapsize::HeapSizeOf;
use memorydb::MemoryDB;
use patricia_trie::{TrieDB, TrieError, Trie, NodeCodec};
use trie::{TrieDB, Trie, MemoryDB, DBValue, TrieError};
use changes_trie::Storage as ChangesTrieStorage;
/// Patricia trie-based storage trait.
@@ -34,19 +32,17 @@ pub trait Storage<H: Hasher>: Send + Sync {
}
/// Patricia trie-based pairs storage essence.
pub struct TrieBackendEssence<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> {
pub struct TrieBackendEssence<S: TrieBackendStorage<H>, H: Hasher> {
storage: S,
root: H::Out,
_codec: PhantomData<C>,
}
impl<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> TrieBackendEssence<S, H, C> where H::Out: HeapSizeOf {
impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> where H::Out: HeapSizeOf {
/// Create new trie-based backend.
pub fn new(storage: S, root: H::Out) -> Self {
TrieBackendEssence {
storage,
root,
_codec: Default::default(),
}
}
@@ -70,7 +66,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> TrieBackendEssence<S,
let map_e = |e| format!("Trie lookup error: {}", e);
TrieDB::<H, C>::new(&eph, &self.root).map_err(map_e)?
TrieDB::<H>::new(&eph, &self.root).map_err(map_e)?
.get(key).map(|x| x.map(|val| val.to_vec())).map_err(map_e)
}
@@ -82,8 +78,8 @@ impl<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> TrieBackendEssence<S,
overlay: &mut read_overlay,
};
let mut iter = move || -> Result<(), Box<TrieError<H::Out, C::Error>>> {
let trie = TrieDB::<H, C>::new(&eph, &self.root)?;
let mut iter = move || -> Result<(), Box<TrieError<H::Out>>> {
let trie = TrieDB::<H>::new(&eph, &self.root)?;
let mut iter = trie.iter()?;
iter.seek(prefix)?;
@@ -112,12 +108,18 @@ pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> {
overlay: &'a mut MemoryDB<H>,
}
impl<'a, S: TrieBackendStorage<H>, H: Hasher> AsHashDB<H> for Ephemeral<'a, S, H> where H::Out: HeapSizeOf {
fn as_hashdb(&self) -> &HashDB<H> { self }
fn as_hashdb_mut(&mut self) -> &mut HashDB<H> { self }
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: 'a + Hasher
> hash_db::AsHashDB<H, DBValue>
for Ephemeral<'a, S, H>
where H::Out: HeapSizeOf
{
fn as_hash_db<'b>(&'b self) -> &'b (hash_db::HashDB<H, DBValue> + 'b) { self }
fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (hash_db::HashDB<H, DBValue> + 'b) { self }
}
impl<'a, S: 'a + TrieBackendStorage<H>, H: Hasher> Ephemeral<'a, S, H> {
impl<'a, S: TrieBackendStorage<H>, H: Hasher> Ephemeral<'a, S, H> {
pub fn new(storage: &'a S, overlay: &'a mut MemoryDB<H>) -> Self {
Ephemeral {
storage,
@@ -126,7 +128,13 @@ impl<'a, S: 'a + TrieBackendStorage<H>, H: Hasher> Ephemeral<'a, S, H> {
}
}
impl<'a, S: TrieBackendStorage<H>, H: Hasher> HashDB<H> for Ephemeral<'a, S, H> where H::Out: HeapSizeOf {
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: Hasher
> hash_db::HashDB<H, DBValue>
for Ephemeral<'a, S, H>
where H::Out: HeapSizeOf
{
fn keys(&self) -> HashMap<H::Out, i32> {
self.overlay.keys() // TODO: iterate backing
}
@@ -137,7 +145,7 @@ impl<'a, S: TrieBackendStorage<H>, H: Hasher> HashDB<H> for Ephemeral<'a, S, H>
if i <= 0 {
None
} else {
Some(val)
Some(val.clone())
}
}
None => match self.storage.get(&key) {
@@ -182,7 +190,7 @@ impl<H: Hasher> TrieBackendStorage<H> for Arc<Storage<H>> {
// This implementation is used by test storage trie clients.
impl<H: Hasher> TrieBackendStorage<H> for MemoryDB<H> {
fn get(&self, key: &H::Out) -> Result<Option<DBValue>, String> {
Ok(HashDB::<H>::get(self, key))
Ok(<Self as hash_db::HashDB<H, DBValue>>::get(self, key))
}
}