Fix key collision for child trie (#4162)

* In progress, runtime io must switch to future proof root +
child_specific (unique id) + u32 type.

* Switch interface, sr-io seems ok, rpc could use similar interface to
sr-io, genesis json broken if there is child trie in existing encoding
genesis.

* test from previous implementation.

* fix proving test.

* Restore Keyspacedb from other branch, only apply to child trie.

* Removing unneeded child_info from child root (child info are stored
if things changed, otherwhise the root does not change).

* Switch rpc to use same format as ext: more future proof.

* use root from child info for trie backend essence.

* Breaking long lines.

* Update doc and clean pr a bit.

* fix error type

* Restore removed doc on merge and update sr-io doc.

* Switch child storage api to use directly unique id, if managed id
where to be put in place, the api will change at this time.

* Clean deprecated host interface from child.

* Removing assertion on child info (can fail depending on root
memoization).

* merging child info in the overlay when possible.

* child iteration by prefix using child_info.

* Using ChainInfo in frame support. ChainInfo gets redesign to avoid
buffers allocation on every calls.

* Add length of root to the data of child info.

* comments

* Encode compact.

* Remove child info with root.

* Fix try_update condition.

* Comment Ext child root caching.

* Replace tuples by struct with field

* remove StorageTuple alias.

* Fix doc tests, and remove StorageOverlay and ChildStorageOverlay
aliases.
This commit is contained in:
cheme
2019-12-14 03:11:19 +01:00
committed by Gavin Wood
parent 7121837f84
commit 0ece5d9e17
53 changed files with 2121 additions and 918 deletions
+191 -74
View File
@@ -26,6 +26,7 @@ use trie::{
trie_types::{TrieDBMut, Layout},
};
use codec::{Encode, Codec};
use primitives::storage::{ChildInfo, OwnedChildInfo, Storage};
/// A state backend is used to read state data and can have changes committed
/// to it.
@@ -50,11 +51,21 @@ pub trait Backend<H: Hasher>: std::fmt::Debug {
}
/// Get keyed child storage or None if there is nothing associated.
fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error>;
fn child_storage(
&self,
storage_key: &[u8],
child_info: ChildInfo,
key: &[u8],
) -> Result<Option<Vec<u8>>, Self::Error>;
/// Get child keyed storage value hash or None if there is nothing associated.
fn child_storage_hash(&self, storage_key: &[u8], key: &[u8]) -> Result<Option<H::Out>, Self::Error> {
self.child_storage(storage_key, key).map(|v| v.map(|v| H::hash(&v)))
fn child_storage_hash(
&self,
storage_key: &[u8],
child_info: ChildInfo,
key: &[u8],
) -> Result<Option<H::Out>, Self::Error> {
self.child_storage(storage_key, child_info, key).map(|v| v.map(|v| H::hash(&v)))
}
/// true if a key exists in storage.
@@ -63,8 +74,13 @@ pub trait Backend<H: Hasher>: std::fmt::Debug {
}
/// true if a key exists in child storage.
fn exists_child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result<bool, Self::Error> {
Ok(self.child_storage(storage_key, key)?.is_some())
fn exists_child_storage(
&self,
storage_key: &[u8],
child_info: ChildInfo,
key: &[u8],
) -> Result<bool, Self::Error> {
Ok(self.child_storage(storage_key, child_info, key)?.is_some())
}
/// Return the next key in storage in lexicographic order or `None` if there is no value.
@@ -74,11 +90,17 @@ pub trait Backend<H: Hasher>: std::fmt::Debug {
fn next_child_storage_key(
&self,
storage_key: &[u8],
child_info: ChildInfo,
key: &[u8]
) -> Result<Option<Vec<u8>>, Self::Error>;
/// Retrieve all entries keys of child storage and call `f` for each of those keys.
fn for_keys_in_child_storage<F: FnMut(&[u8])>(&self, storage_key: &[u8], f: F);
fn for_keys_in_child_storage<F: FnMut(&[u8])>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
f: F,
);
/// Retrieve all entries keys which start with the given prefix and
/// call `f` for each of those keys.
@@ -93,7 +115,13 @@ pub trait Backend<H: Hasher>: std::fmt::Debug {
/// Retrieve all child entries keys which start with the given prefix and
/// call `f` for each of those keys.
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(&self, storage_key: &[u8], prefix: &[u8], f: F);
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
prefix: &[u8],
f: F,
);
/// Calculate the storage root, with given delta over what is already stored in
/// the backend, and produce a "transaction" that can be used to commit.
@@ -106,7 +134,12 @@ pub trait Backend<H: Hasher>: std::fmt::Debug {
/// Calculate the child storage root, with given delta over what is already stored in
/// the backend, and produce a "transaction" that can be used to commit. The second argument
/// is true if child storage root equals default storage root.
fn child_storage_root<I>(&self, storage_key: &[u8], delta: I) -> (H::Out, bool, Self::Transaction)
fn child_storage_root<I>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
delta: I,
) -> (H::Out, bool, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
H::Out: Ord;
@@ -122,9 +155,14 @@ pub trait Backend<H: Hasher>: std::fmt::Debug {
}
/// Get all keys of child storage with given prefix
fn child_keys(&self, child_storage_key: &[u8], prefix: &[u8]) -> Vec<Vec<u8>> {
fn child_keys(
&self,
storage_key: &[u8],
child_info: ChildInfo,
prefix: &[u8],
) -> Vec<Vec<u8>> {
let mut all = Vec::new();
self.for_child_keys_with_prefix(child_storage_key, prefix, |k| all.push(k.to_vec()));
self.for_child_keys_with_prefix(storage_key, child_info, prefix, |k| all.push(k.to_vec()));
all
}
@@ -144,15 +182,15 @@ pub trait Backend<H: Hasher>: std::fmt::Debug {
where
I1: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
I2i: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
I2: IntoIterator<Item=(Vec<u8>, I2i)>,
I2: IntoIterator<Item=(Vec<u8>, I2i, OwnedChildInfo)>,
H::Out: Ord + Encode,
{
let mut txs: Self::Transaction = Default::default();
let mut child_roots: Vec<_> = Default::default();
// child first
for (storage_key, child_delta) in child_deltas {
for (storage_key, child_delta, child_info) in child_deltas {
let (child_root, empty, child_txs) =
self.child_storage_root(&storage_key[..], child_delta);
self.child_storage_root(&storage_key[..], child_info.as_ref(), child_delta);
txs.consolidate(child_txs);
if empty {
child_roots.push((storage_key, None));
@@ -177,28 +215,49 @@ impl<'a, T: Backend<H>, H: Hasher> Backend<H> for &'a T {
(*self).storage(key)
}
fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
(*self).child_storage(storage_key, key)
fn child_storage(
&self,
storage_key: &[u8],
child_info: ChildInfo,
key: &[u8],
) -> Result<Option<Vec<u8>>, Self::Error> {
(*self).child_storage(storage_key, child_info, key)
}
fn for_keys_in_child_storage<F: FnMut(&[u8])>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
f: F,
) {
(*self).for_keys_in_child_storage(storage_key, child_info, f)
}
fn next_storage_key(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
(*self).next_storage_key(key)
}
fn next_child_storage_key(&self, storage_key: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
(*self).next_child_storage_key(storage_key, key)
}
fn for_keys_in_child_storage<F: FnMut(&[u8])>(&self, storage_key: &[u8], f: F) {
(*self).for_keys_in_child_storage(storage_key, f)
fn next_child_storage_key(
&self,
storage_key: &[u8],
child_info: ChildInfo,
key: &[u8],
) -> Result<Option<Vec<u8>>, Self::Error> {
(*self).next_child_storage_key(storage_key, child_info, key)
}
fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
(*self).for_keys_with_prefix(prefix, f)
}
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(&self, storage_key: &[u8], prefix: &[u8], f: F) {
(*self).for_child_keys_with_prefix(storage_key, prefix, f)
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
prefix: &[u8],
f: F,
) {
(*self).for_child_keys_with_prefix(storage_key, child_info, prefix, f)
}
fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
@@ -209,12 +268,17 @@ impl<'a, T: Backend<H>, H: Hasher> Backend<H> for &'a T {
(*self).storage_root(delta)
}
fn child_storage_root<I>(&self, storage_key: &[u8], delta: I) -> (H::Out, bool, Self::Transaction)
fn child_storage_root<I>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
delta: I,
) -> (H::Out, bool, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
H::Out: Ord,
{
(*self).child_storage_root(storage_key, delta)
(*self).child_storage_root(storage_key, child_info, delta)
}
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
@@ -238,7 +302,10 @@ impl Consolidate for () {
}
}
impl Consolidate for Vec<(Option<Vec<u8>>, Vec<u8>, Option<Vec<u8>>)> {
impl Consolidate for Vec<(
Option<(Vec<u8>, OwnedChildInfo)>,
Vec<(Vec<u8>, Option<Vec<u8>>)>,
)> {
fn consolidate(&mut self, mut other: Self) {
self.append(&mut other);
}
@@ -268,7 +335,7 @@ impl error::Error for Void {
/// In-memory backend. Fully recomputes tries each time `as_trie_backend` is called but useful for
/// tests and proof checking.
pub struct InMemory<H: Hasher> {
inner: HashMap<Option<Vec<u8>>, BTreeMap<Vec<u8>, Vec<u8>>>,
inner: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>,
// This field is only needed for returning reference in `as_trie_backend`.
trie: Option<TrieBackend<MemoryDB<H>, H>>,
_hasher: PhantomData<H>,
@@ -310,19 +377,21 @@ impl<H: Hasher> InMemory<H> where H::Out: Codec {
/// Copy the state, with applied updates
pub fn update(&self, changes: <Self as Backend<H>>::Transaction) -> Self {
let mut inner = self.inner.clone();
for (storage_key, key, val) in changes {
match val {
Some(v) => { inner.entry(storage_key).or_default().insert(key, v); },
None => { inner.entry(storage_key).or_default().remove(&key); },
for (child_info, key_values) in changes {
let entry = inner.entry(child_info).or_default();
for (key, val) in key_values {
match val {
Some(v) => { entry.insert(key, v); },
None => { entry.remove(&key); },
}
}
}
inner.into()
}
}
impl<H: Hasher> From<HashMap<Option<Vec<u8>>, BTreeMap<Vec<u8>, Vec<u8>>>> for InMemory<H> {
fn from(inner: HashMap<Option<Vec<u8>>, BTreeMap<Vec<u8>, Vec<u8>>>) -> Self {
impl<H: Hasher> From<HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>> for InMemory<H> {
fn from(inner: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>) -> Self {
InMemory {
inner: inner,
trie: None,
@@ -331,17 +400,11 @@ impl<H: Hasher> From<HashMap<Option<Vec<u8>>, BTreeMap<Vec<u8>, Vec<u8>>>> for I
}
}
impl<H: Hasher> From<(
BTreeMap<Vec<u8>, Vec<u8>>,
HashMap<Vec<u8>, BTreeMap<Vec<u8>, Vec<u8>>>,
)> for InMemory<H> {
fn from(inners: (
BTreeMap<Vec<u8>, Vec<u8>>,
HashMap<Vec<u8>, BTreeMap<Vec<u8>, Vec<u8>>>,
)) -> Self {
let mut inner: HashMap<Option<Vec<u8>>, BTreeMap<Vec<u8>, Vec<u8>>>
= inners.1.into_iter().map(|(k, v)| (Some(k), v)).collect();
inner.insert(None, inners.0);
impl<H: Hasher> From<Storage> for InMemory<H> {
fn from(inners: Storage) -> Self {
let mut inner: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>
= inners.children.into_iter().map(|(k, c)| (Some((k, c.child_info)), c.data)).collect();
inner.insert(None, inners.top);
InMemory {
inner: inner,
trie: None,
@@ -362,12 +425,19 @@ impl<H: Hasher> From<BTreeMap<Vec<u8>, Vec<u8>>> for InMemory<H> {
}
}
impl<H: Hasher> From<Vec<(Option<Vec<u8>>, Vec<u8>, Option<Vec<u8>>)>> for InMemory<H> {
fn from(inner: Vec<(Option<Vec<u8>>, Vec<u8>, Option<Vec<u8>>)>) -> Self {
let mut expanded: HashMap<Option<Vec<u8>>, BTreeMap<Vec<u8>, Vec<u8>>> = HashMap::new();
for (child_key, key, value) in inner {
if let Some(value) = value {
expanded.entry(child_key).or_default().insert(key, value);
impl<H: Hasher> From<Vec<(Option<(Vec<u8>, OwnedChildInfo)>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>>
for InMemory<H> {
fn from(
inner: Vec<(Option<(Vec<u8>, OwnedChildInfo)>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>,
) -> Self {
let mut expanded: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>
= HashMap::new();
for (child_info, key_values) in inner {
let entry = expanded.entry(child_info).or_default();
for (key, value) in key_values {
if let Some(value) = value {
entry.insert(key, value);
}
}
}
expanded.into()
@@ -376,22 +446,33 @@ impl<H: Hasher> From<Vec<(Option<Vec<u8>>, Vec<u8>, Option<Vec<u8>>)>> for InMem
impl<H: Hasher> InMemory<H> {
/// child storage key iterator
pub fn child_storage_keys(&self) -> impl Iterator<Item=&[u8]> {
self.inner.iter().filter_map(|item| item.0.as_ref().map(|v|&v[..]))
pub fn child_storage_keys(&self) -> impl Iterator<Item=(&[u8], ChildInfo)> {
self.inner.iter().filter_map(|item|
item.0.as_ref().map(|v|(&v.0[..], v.1.as_ref()))
)
}
}
impl<H: Hasher> Backend<H> for InMemory<H> where H::Out: Codec {
type Error = Void;
type Transaction = Vec<(Option<Vec<u8>>, Vec<u8>, Option<Vec<u8>>)>;
type Transaction = Vec<(
Option<(Vec<u8>, OwnedChildInfo)>,
Vec<(Vec<u8>, Option<Vec<u8>>)>,
)>;
type TrieBackendStorage = MemoryDB<H>;
fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
Ok(self.inner.get(&None).and_then(|map| map.get(key).map(Clone::clone)))
}
fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
Ok(self.inner.get(&Some(storage_key.to_vec())).and_then(|map| map.get(key).map(Clone::clone)))
fn child_storage(
&self,
storage_key: &[u8],
child_info: ChildInfo,
key: &[u8],
) -> Result<Option<Vec<u8>>, Self::Error> {
Ok(self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
.and_then(|map| map.get(key).map(Clone::clone)))
}
fn exists_storage(&self, key: &[u8]) -> Result<bool, Self::Error> {
@@ -406,9 +487,14 @@ impl<H: Hasher> Backend<H> for InMemory<H> where H::Out: Codec {
Ok(next_key)
}
fn next_child_storage_key(&self, storage_key: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
fn next_child_storage_key(
&self,
storage_key: &[u8],
child_info: ChildInfo,
key: &[u8],
) -> Result<Option<Vec<u8>>, Self::Error> {
let range = (ops::Bound::Excluded(key), ops::Bound::Unbounded);
let next_key = self.inner.get(&Some(storage_key.to_vec()))
let next_key = self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
.and_then(|map| map.range::<[u8], _>(range).next().map(|(k, _)| k).cloned());
Ok(next_key)
@@ -423,12 +509,24 @@ impl<H: Hasher> Backend<H> for InMemory<H> where H::Out: Codec {
.for_each(|(k, v)| f(k, v)));
}
fn for_keys_in_child_storage<F: FnMut(&[u8])>(&self, storage_key: &[u8], mut f: F) {
self.inner.get(&Some(storage_key.to_vec())).map(|map| map.keys().for_each(|k| f(&k)));
fn for_keys_in_child_storage<F: FnMut(&[u8])>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
mut f: F,
) {
self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
.map(|map| map.keys().for_each(|k| f(&k)));
}
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(&self, storage_key: &[u8], prefix: &[u8], f: F) {
self.inner.get(&Some(storage_key.to_vec()))
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
prefix: &[u8],
f: F,
) {
self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
.map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f));
}
@@ -448,19 +546,26 @@ impl<H: Hasher> Backend<H> for InMemory<H> where H::Out: Codec {
.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
);
let full_transaction = transaction.into_iter().map(|(k, v)| (None, k, v)).collect();
let full_transaction = transaction.into_iter().collect();
(root, full_transaction)
(root, vec![(None, full_transaction)])
}
fn child_storage_root<I>(&self, storage_key: &[u8], delta: I) -> (H::Out, bool, Self::Transaction)
fn child_storage_root<I>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
delta: I,
) -> (H::Out, bool, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
H::Out: Ord
{
let storage_key = storage_key.to_vec();
let child_info = Some((storage_key.clone(), child_info.to_owned()));
let existing_pairs = self.inner.get(&Some(storage_key.clone()))
let existing_pairs = self.inner.get(&child_info)
.into_iter()
.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
@@ -473,11 +578,11 @@ impl<H: Hasher> Backend<H> for InMemory<H> where H::Out: Codec {
.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
);
let full_transaction = transaction.into_iter().map(|(k, v)| (Some(storage_key.clone()), k, v)).collect();
let full_transaction = transaction.into_iter().collect();
let is_default = root == default_child_trie_root::<Layout<H>>(&storage_key);
(root, is_default, full_transaction)
(root, is_default, vec![(child_info, full_transaction)])
}
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
@@ -494,8 +599,13 @@ impl<H: Hasher> Backend<H> for InMemory<H> where H::Out: Codec {
.collect()
}
fn child_keys(&self, storage_key: &[u8], prefix: &[u8]) -> Vec<Vec<u8>> {
self.inner.get(&Some(storage_key.to_vec()))
fn child_keys(
&self,
storage_key: &[u8],
child_info: ChildInfo,
prefix: &[u8],
) -> Vec<Vec<u8>> {
self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
.into_iter()
.flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned())
.collect()
@@ -505,8 +615,10 @@ impl<H: Hasher> Backend<H> for InMemory<H> where H::Out: Codec {
let mut mdb = MemoryDB::default();
let mut new_child_roots = Vec::new();
let mut root_map = None;
for (storage_key, map) in &self.inner {
if let Some(storage_key) = storage_key.as_ref() {
for (child_info, map) in &self.inner {
if let Some((storage_key, _child_info)) = child_info.as_ref() {
// no need to use child_info at this point because we use a MemoryDB for
// proof (with PrefixedMemoryDB it would be needed).
let ch = insert_into_memory_db::<H, _>(&mut mdb, map.clone().into_iter())?;
new_child_roots.push((storage_key.clone(), ch.as_ref().into()));
} else {
@@ -556,11 +668,16 @@ mod tests {
#[test]
fn in_memory_with_child_trie_only() {
let storage = InMemory::<primitives::Blake2Hasher>::default();
let child_info = OwnedChildInfo::new_default(b"unique_id_1".to_vec());
let mut storage = storage.update(
vec![(Some(b"1".to_vec()), b"2".to_vec(), Some(b"3".to_vec()))]
vec![(
Some((b"1".to_vec(), child_info.clone())),
vec![(b"2".to_vec(), Some(b"3".to_vec()))]
)]
);
let trie_backend = storage.as_trie_backend().unwrap();
assert_eq!(trie_backend.child_storage(b"1", b"2").unwrap(), Some(b"3".to_vec()));
assert_eq!(trie_backend.child_storage(b"1", child_info.as_ref(), b"2").unwrap(),
Some(b"3".to_vec()));
assert!(trie_backend.storage(b"1").unwrap().is_some());
}
}