pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-17 05:31:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support multi-hash in multi-trie via PlainDB (#1106)

Browse Source 
* Temporarily pin trie to #2

* Use generic and delay trait object casting

Rust does not support super-trait upcasting

* Add PlainDB impl for Ephemeral

* Add PlainDB trait alias for completeness

* Use PlainDB for test TrieBackendStorage fetch

We always check overlay first for a storage fetch, which already checked null data. Using PlainDB here makes it work
nicer with other PlainDB overlays.

* Update trie reference

* Use HashDBRef in places when approriate

* Use PlainDBRef in places when approriate

* Update trie crate reference

* Remove unused HashDB::keys

* Patch dependencies

* Fix cargolock

* Update cargo lock again
This commit is contained in:
Wei Tang
2019-02-06 11:16:40 +01:00
committed by Arkadiy Paronyan
parent fa2e323478
commit 1ba7e35c18
11 changed files with 322 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/core/executor/wasm/Cargo.lock
Generated
+39 -4
View File
@@ -34,7 +34,7 @@ dependencies = [
[[package]]
name = "hash-db"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
source = "git+https://github.com/paritytech/trie#696a380c1bf1a816a0c1872a6349e4b12e19b3ef"
[[package]]
name = "hash256-std-hasher"
@@ -145,7 +145,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
name = "sr-io"
version = "0.1.0"
dependencies = [
"hash-db 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"hash-db 0.9.0 (git+https://github.com/paritytech/trie)",
"parity-codec 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"sr-std 0.1.0",
@@ -179,7 +179,7 @@ name = "substrate-primitives"
version = "0.1.0"
dependencies = [
"byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
"hash-db 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"hash-db 0.9.0 (git+https://github.com/paritytech/trie)",
"hash256-std-hasher 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-codec 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-codec-derive 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -214,13 +214,48 @@ name = "unicode-xid"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[patch.unused]]
name = "hash256-std-hasher"
version = "0.9.1"
source = "git+https://github.com/paritytech/trie#696a380c1bf1a816a0c1872a6349e4b12e19b3ef"
[[patch.unused]]
name = "keccak-hasher"
version = "0.2.1"
source = "git+https://github.com/paritytech/trie#696a380c1bf1a816a0c1872a6349e4b12e19b3ef"
[[patch.unused]]
name = "memory-db"
version = "0.9.1"
source = "git+https://github.com/paritytech/trie#696a380c1bf1a816a0c1872a6349e4b12e19b3ef"
[[patch.unused]]
name = "trie-bench"
version = "0.10.0"
source = "git+https://github.com/paritytech/trie#696a380c1bf1a816a0c1872a6349e4b12e19b3ef"
[[patch.unused]]
name = "trie-db"
version = "0.9.1"
source = "git+https://github.com/paritytech/trie#696a380c1bf1a816a0c1872a6349e4b12e19b3ef"
[[patch.unused]]
name = "trie-root"
version = "0.9.1"
source = "git+https://github.com/paritytech/trie#696a380c1bf1a816a0c1872a6349e4b12e19b3ef"
[[patch.unused]]
name = "trie-standardmap"
version = "0.9.1"
source = "git+https://github.com/paritytech/trie#696a380c1bf1a816a0c1872a6349e4b12e19b3ef"
[metadata]
"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"
"checksum byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "90492c5858dd7d2e78691cfb89f90d273a2800fc11d98f60786e5d87e2f83781"
"checksum crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a2f4a431c5c9f662e1200b7c7f02c34e91361150e382089a8f2dec3ba680cbda"
"checksum crunchy 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c240f247c278fa08a6d4820a6a222bfc6e0d999e51ba67be94f44c905b2161f2"
"checksum fixed-hash 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a557e80084b05c32b455963ff565a9de6f2866da023d6671705c6aff6f65e01c"
"checksum hash-db 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "dc5ec43724866bbc8337e09cab4d4b5f9fdbbe589f04bdc8bfda906a639ad338"
"checksum hash-db 0.9.0 (git+https://github.com/paritytech/trie)" = "<none>"
"checksum hash256-std-hasher 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "811bd8c26961527b7d5623b71162d865325639f8ca204d4ec90b5b87473a122d"
"checksum impl-codec 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2050d823639fbeae26b2b5ba09aca8907793117324858070ade0673c49f793b"
"checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2"
substrate/core/executor/wasm/Cargo.toml
+10
View File
@@ -18,3 +18,13 @@ lto = true
[workspace]
members = []
[patch.crates-io]
hash-db = { git = "https://github.com/paritytech/trie" }
hash256-std-hasher = { git = "https://github.com/paritytech/trie" }
keccak-hasher = { git = "https://github.com/paritytech/trie" }
memory-db = { git = "https://github.com/paritytech/trie" }
trie-bench = { git = "https://github.com/paritytech/trie" }
trie-db = { git = "https://github.com/paritytech/trie" }
trie-root = { git = "https://github.com/paritytech/trie" }
trie-standardmap = { git = "https://github.com/paritytech/trie" }
substrate/core/state-machine/src/proving_backend.rs
+2 -2
View File
@@ -48,7 +48,7 @@ impl<'a, S, H> ProvingBackendEssence<'a, S, H>
let map_e = |e| format!("Trie lookup error: {}", e);
read_trie_value_with(&eph, self.backend.root(), key, &mut *self.proof_recorder).map_err(map_e)
read_trie_value_with::<H, _, Ephemeral<S, H>>(&eph, self.backend.root(), key, &mut *self.proof_recorder).map_err(map_e)
}
pub fn child_storage(&mut self, storage_key: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, String> {
@@ -74,7 +74,7 @@ impl<'a, S, H> ProvingBackendEssence<'a, S, H>
let mut iter = move || -> Result<(), Box<TrieError<H::Out>>> {
let root = self.backend.root();
record_all_keys::<H>(&eph, root, &mut *self.proof_recorder)
record_all_keys::<H, _>(&eph, root, &mut *self.proof_recorder)
};
if let Err(e) = iter() {
substrate/core/state-machine/src/trie_backend.rs
+2 -2
View File
@@ -138,7 +138,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher> Backend<H> for TrieBackend<S, H> where
&mut write_overlay,
);
match delta_trie_root::<H, _, _, _>(&mut eph, root, delta) {
match delta_trie_root::<H, _, _, _, _>(&mut eph, root, delta) {
Ok(ret) => root = ret,
Err(e) => warn!(target: "trie", "Failed to write to trie: {}", e),
}
@@ -169,7 +169,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher> Backend<H> for TrieBackend<S, H> where
&mut write_overlay,
);
match child_delta_trie_root::<H, _, _, _>(storage_key, &mut eph, root.clone(), delta) {
match child_delta_trie_root::<H, _, _, _, _>(storage_key, &mut eph, root.clone(), delta) {
Ok(ret) => root = ret,
Err(e) => warn!(target: "trie", "Failed to write to trie: {}", e),
}
substrate/core/state-machine/src/trie_backend_essence.rs
+82 -25
View File
@@ -17,7 +17,6 @@
//! Trie-based state machine backend essence used to read values
//! from storage.
use std::collections::HashMap;
use std::ops::Deref;
use std::sync::Arc;
use log::{debug, warn};
@@ -106,7 +105,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> where H::Out:
overlay: &mut read_overlay,
};
if let Err(e) = for_keys_in_child_trie::<H, _>(storage_key, &eph, &root, f) {
if let Err(e) = for_keys_in_child_trie::<H, _, Ephemeral<S, H>>(storage_key, &eph, &root, f) {
debug!(target: "trie", "Error while iterating child storage: {}", e);
}
}
@@ -149,6 +148,17 @@ pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> {
overlay: &'a mut MemoryDB<H>,
}
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: 'a + Hasher
> hash_db::AsPlainDB<H::Out, DBValue>
for Ephemeral<'a, S, H>
where H::Out: HeapSizeOf
{
fn as_plain_db<'b>(&'b self) -> &'b (hash_db::PlainDB<H::Out, DBValue> + 'b) { self }
fn as_plain_db_mut<'b>(&'b mut self) -> &'b mut (hash_db::PlainDB<H::Out, DBValue> + 'b) { self }
}
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: 'a + Hasher
@@ -172,50 +182,97 @@ impl<'a, S: TrieBackendStorage<H>, H: Hasher> Ephemeral<'a, S, H> {
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: Hasher
> hash_db::HashDB<H, DBValue>
> hash_db::PlainDB<H::Out, DBValue>
for Ephemeral<'a, S, H>
where H::Out: HeapSizeOf
{
fn keys(&self) -> HashMap<H::Out, i32> {
self.overlay.keys()
}
fn get(&self, key: &H::Out) -> Option<DBValue> {
match self.overlay.raw(key) {
Some((val, i)) => {
if i <= 0 {
None
} else {
Some(val.clone())
}
}
None => match self.storage.get(&key) {
if let Some(val) = hash_db::PlainDB::get(self.overlay, key) {
Some(val)
} else {
match self.storage.get(&key) {
Ok(x) => x,
Err(e) => {
warn!(target: "trie", "Failed to read from DB: {}", e);
None
},
},
}
}
}
fn contains(&self, key: &H::Out) -> bool {
self.get(key).is_some()
}
fn insert(&mut self, value: &[u8]) -> H::Out {
self.overlay.insert(value)
hash_db::PlainDB::get(self, key).is_some()
}
fn emplace(&mut self, key: H::Out, value: DBValue) {
self.overlay.emplace(key, value)
hash_db::PlainDB::emplace(self.overlay, key, value)
}
fn remove(&mut self, key: &H::Out) {
self.overlay.remove(key)
hash_db::PlainDB::remove(self.overlay, key)
}
}
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: Hasher
> hash_db::PlainDBRef<H::Out, DBValue>
for Ephemeral<'a, S, H>
where H::Out: HeapSizeOf
{
fn get(&self, key: &H::Out) -> Option<DBValue> { hash_db::PlainDB::get(self, key) }
fn contains(&self, key: &H::Out) -> bool { hash_db::PlainDB::contains(self, key) }
}
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: Hasher
> hash_db::HashDB<H, DBValue>
for Ephemeral<'a, S, H>
where H::Out: HeapSizeOf
{
fn get(&self, key: &H::Out) -> Option<DBValue> {
if let Some(val) = hash_db::HashDB::get(self.overlay, key) {
Some(val)
} else {
match self.storage.get(&key) {
Ok(x) => x,
Err(e) => {
warn!(target: "trie", "Failed to read from DB: {}", e);
None
},
}
}
}
fn contains(&self, key: &H::Out) -> bool {
hash_db::HashDB::get(self, key).is_some()
}
fn insert(&mut self, value: &[u8]) -> H::Out {
hash_db::HashDB::insert(self.overlay, value)
}
fn emplace(&mut self, key: H::Out, value: DBValue) {
hash_db::HashDB::emplace(self.overlay, key, value)
}
fn remove(&mut self, key: &H::Out) {
hash_db::HashDB::remove(self.overlay, key)
}
}
impl<'a,
S: 'a + TrieBackendStorage<H>,
H: Hasher
> hash_db::HashDBRef<H, DBValue>
for Ephemeral<'a, S, H>
where H::Out: HeapSizeOf
{
fn get(&self, key: &H::Out) -> Option<DBValue> { hash_db::HashDB::get(self, key) }
fn contains(&self, key: &H::Out) -> bool { hash_db::HashDB::contains(self, key) }
}
/// Key-value pairs storage that is used by trie backend essence.
pub trait TrieBackendStorage<H: Hasher>: Send + Sync {
/// Get the value stored at key.
@@ -232,7 +289,7 @@ impl<H: Hasher> TrieBackendStorage<H> for Arc<Storage<H>> {
// This implementation is used by test storage trie clients.
impl<H: Hasher> TrieBackendStorage<H> for MemoryDB<H> {
fn get(&self, key: &H::Out) -> Result<Option<DBValue>, String> {
Ok(<Self as hash_db::HashDB<H, DBValue>>::get(self, key))
Ok(hash_db::PlainDB::get(self, key))
}
}
substrate/core/test-runtime/Cargo.toml
+10
View File
@@ -44,3 +44,13 @@ std = [
"runtime_version/std",
"consensus_aura/std",
]
[patch.crates-io]
hash-db = { git = "https://github.com/paritytech/trie" }
hash256-std-hasher = { git = "https://github.com/paritytech/trie" }
keccak-hasher = { git = "https://github.com/paritytech/trie" }
memory-db = { git = "https://github.com/paritytech/trie" }
trie-bench = { git = "https://github.com/paritytech/trie" }
trie-db = { git = "https://github.com/paritytech/trie" }
trie-root = { git = "https://github.com/paritytech/trie" }
trie-standardmap = { git = "https://github.com/paritytech/trie" }
substrate/core/trie/src/lib.rs
+66 -16
View File
@@ -40,6 +40,8 @@ pub trait AsHashDB<H: Hasher>: hash_db::AsHashDB<H, trie_db::DBValue> {}
impl<H: Hasher, T: hash_db::AsHashDB<H, trie_db::DBValue>> AsHashDB<H> for T {}
/// As in `hash_db`, but less generic, trait exposed.
pub type HashDB<'a, H> = hash_db::HashDB<H, trie_db::DBValue> + 'a;
/// As in `hash_db`, but less generic, trait exposed.
pub type PlainDB<'a, K> = hash_db::PlainDB<K, trie_db::DBValue> + 'a;
/// As in `memory_db`, but less generic, trait exposed.
pub type MemoryDB<H> = memory_db::MemoryDB<H, trie_db::DBValue>;
@@ -60,13 +62,18 @@ pub fn trie_root<H: Hasher, I, A, B>(input: I) -> H::Out where
}
/// Determine a trie root given a hash DB and delta values.
pub fn delta_trie_root<H: Hasher, I, A, B>(db: &mut HashDB<H>, mut root: H::Out, delta: I) -> Result<H::Out, Box<TrieError<H::Out>>> where
pub fn delta_trie_root<H: Hasher, I, A, B, DB>(
db: &mut DB,
mut root: H::Out,
delta: I
) -> Result<H::Out, Box<TrieError<H::Out>>> where
I: IntoIterator<Item = (A, Option<B>)>,
A: AsRef<[u8]> + Ord,
B: AsRef<[u8]>,
DB: hash_db::HashDB<H, trie_db::DBValue>,
{
{
let mut trie = TrieDBMut::<H>::from_existing(db, &mut root)?;
let mut trie = TrieDBMut::<H>::from_existing(&mut *db, &mut root)?;
for (key, change) in delta {
match change {
@@ -80,13 +87,22 @@ pub fn delta_trie_root<H: Hasher, I, A, B>(db: &mut HashDB<H>, mut root: H::Out,
}
/// Read a value from the trie.
pub fn read_trie_value<H: Hasher>(db: &HashDB<H>, root: &H::Out, key: &[u8]) -> Result<Option<Vec<u8>>, Box<TrieError<H::Out>>> {
Ok(TrieDB::<H>::new(db, root)?.get(key).map(|x| x.map(|val| val.to_vec()))?)
pub fn read_trie_value<H: Hasher, DB: hash_db::HashDBRef<H, trie_db::DBValue>>(
db: &DB,
root: &H::Out,
key: &[u8]
) -> Result<Option<Vec<u8>>, Box<TrieError<H::Out>>> {
Ok(TrieDB::<H>::new(&*db, root)?.get(key).map(|x| x.map(|val| val.to_vec()))?)
}
/// Read a value from the trie with given Query.
pub fn read_trie_value_with<H: Hasher, Q: Query<H, Item=DBValue>>(db: &HashDB<H>, root: &H::Out, key: &[u8], query: Q) -> Result<Option<Vec<u8>>, Box<TrieError<H::Out>>> {
Ok(TrieDB::<H>::new(db, root)?.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?)
pub fn read_trie_value_with<H: Hasher, Q: Query<H, Item=DBValue>, DB: hash_db::HashDBRef<H, trie_db::DBValue>>(
db: &DB,
root: &H::Out,
key: &[u8],
query: Q
) -> Result<Option<Vec<u8>>, Box<TrieError<H::Out>>> {
Ok(TrieDB::<H>::new(&*db, root)?.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?)
}
/// Determine a trie root node's data given its ordered contents, closed form.
@@ -137,16 +153,22 @@ pub fn child_trie_root<H: Hasher, I, A, B>(_storage_key: &[u8], input: I) -> Vec
}
/// Determine a child trie root given a hash DB and delta values. H is the default hasher, but a generic implementation may ignore this type parameter and use other hashers.
pub fn child_delta_trie_root<H: Hasher, I, A, B>(_storage_key: &[u8], db: &mut HashDB<H>, root_vec: Vec<u8>, delta: I) -> Result<Vec<u8>, Box<TrieError<H::Out>>> where
pub fn child_delta_trie_root<H: Hasher, I, A, B, DB>(
_storage_key: &[u8],
db: &mut DB,
root_vec: Vec<u8>,
delta: I
) -> Result<Vec<u8>, Box<TrieError<H::Out>>> where
I: IntoIterator<Item = (A, Option<B>)>,
A: AsRef<[u8]> + Ord,
B: AsRef<[u8]>,
DB: hash_db::HashDB<H, trie_db::DBValue> + hash_db::PlainDB<H::Out, trie_db::DBValue>,
{
let mut root = H::Out::default();
root.as_mut().copy_from_slice(&root_vec); // root is fetched from DB, not writable by runtime, so it's always valid.
{
let mut trie = TrieDBMut::<H>::from_existing(db, &mut root)?;
let mut trie = TrieDBMut::<H>::from_existing(&mut *db, &mut root)?;
for (key, change) in delta {
match change {
@@ -160,11 +182,18 @@ pub fn child_delta_trie_root<H: Hasher, I, A, B>(_storage_key: &[u8], db: &mut H
}
/// Call `f` for all keys in a child trie.
pub fn for_keys_in_child_trie<H: Hasher, F: FnMut(&[u8])>(_storage_key: &[u8], db: &HashDB<H>, root_slice: &[u8], mut f: F) -> Result<(), Box<TrieError<H::Out>>> {
pub fn for_keys_in_child_trie<H: Hasher, F: FnMut(&[u8]), DB>(
_storage_key: &[u8],
db: &DB,
root_slice: &[u8],
mut f: F
) -> Result<(), Box<TrieError<H::Out>>> where
DB: hash_db::HashDBRef<H, trie_db::DBValue> + hash_db::PlainDBRef<H::Out, trie_db::DBValue>,
{
let mut root = H::Out::default();
root.as_mut().copy_from_slice(root_slice); // root is fetched from DB, not writable by runtime, so it's always valid.
let trie = TrieDB::<H>::new(db, &root)?;
let trie = TrieDB::<H>::new(&*db, &root)?;
let iter = trie.iter()?;
for x in iter {
@@ -176,8 +205,14 @@ pub fn for_keys_in_child_trie<H: Hasher, F: FnMut(&[u8])>(_storage_key: &[u8], d
}
/// Record all keys for a given root.
pub fn record_all_keys<H: Hasher>(db: &HashDB<H>, root: &H::Out, recorder: &mut Recorder<H::Out>) -> Result<(), Box<TrieError<H::Out>>> {
let trie = TrieDB::<H>::new(db, root)?;
pub fn record_all_keys<H: Hasher, DB>(
db: &DB,
root: &H::Out,
recorder: &mut Recorder<H::Out>
) -> Result<(), Box<TrieError<H::Out>>> where
DB: hash_db::HashDBRef<H, trie_db::DBValue>
{
let trie = TrieDB::<H>::new(&*db, root)?;
let iter = trie.iter()?;
for x in iter {
@@ -193,19 +228,34 @@ pub fn record_all_keys<H: Hasher>(db: &HashDB<H>, root: &H::Out, recorder: &mut
}
/// Read a value from the child trie.
pub fn read_child_trie_value<H: Hasher>(_storage_key: &[u8], db: &HashDB<H>, root_slice: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, Box<TrieError<H::Out>>> {
pub fn read_child_trie_value<H: Hasher, DB>(
_storage_key: &[u8],
db: &DB,
root_slice: &[u8],
key: &[u8]
) -> Result<Option<Vec<u8>>, Box<TrieError<H::Out>>> where
DB: hash_db::HashDBRef<H, trie_db::DBValue> + hash_db::PlainDBRef<H::Out, trie_db::DBValue>,
{
let mut root = H::Out::default();
root.as_mut().copy_from_slice(root_slice); // root is fetched from DB, not writable by runtime, so it's always valid.
Ok(TrieDB::<H>::new(db, &root)?.get(key).map(|x| x.map(|val| val.to_vec()))?)
Ok(TrieDB::<H>::new(&*db, &root)?.get(key).map(|x| x.map(|val| val.to_vec()))?)
}
/// Read a value from the child trie with given query.
pub fn read_child_trie_value_with<H: Hasher, Q: Query<H, Item=DBValue>>(_storage_key: &[u8], db: &HashDB<H>, root_slice: &[u8], key: &[u8], query: Q) -> Result<Option<Vec<u8>>, Box<TrieError<H::Out>>> {
pub fn read_child_trie_value_with<H: Hasher, Q: Query<H, Item=DBValue>, DB>(
_storage_key: &[u8],
db: &DB,
root_slice: &[u8],
key: &[u8],
query: Q
) -> Result<Option<Vec<u8>>, Box<TrieError<H::Out>>> where
DB: hash_db::HashDBRef<H, trie_db::DBValue> + hash_db::PlainDBRef<H::Out, trie_db::DBValue>,
{
let mut root = H::Out::default();
root.as_mut().copy_from_slice(root_slice); // root is fetched from DB, not writable by runtime, so it's always valid.
Ok(TrieDB::<H>::new(db, &root)?.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?)
Ok(TrieDB::<H>::new(&*db, &root)?.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?)
}
// Utilities (not exported):