Child storage tests and genesis fix. (#3185)

* Using child storage, (srml-support only), test failing .

* fix simple tests.

* Enumerable by requiring owned struct (previous form only allow
&'static).
Broken tests are from genesis init.

* implement for_child_keys_with_prefix

* indent

* clear_child_prefix fix.

* clear_child_prefix fix 2.

* fix for storage_impl, if/when allowing child and not child this could be
reverted.

* Fix lot of urlinked child genesis, still need to look upon actual
genesis srml module code.
Probably still a lot of broken code needing debugging.

* switch well_known_key to their associated module child trie.
Fix a genesis init (balance).
Complete some testing.
Comment some tests before using.

* fixing test runtime child keys

* latest commit fix broken genesis init

* fix system balances child name.

* Important fix: storage_root from test externalities need children (it is
already the case for ext).

* executive root with child calculation

* Avoid empty trie on test ext.

* Symetric removal of key for system.

* commenting changes related tests.

* Remove child module specifics.

* fix issues.

* fix some formatting

* fix bench and bump runtime

* Remove extend_storage_overlays, assimilate_storage do the same as is
proper considering srml macro.

* Fix warning for assimilate.

* Removing kill as they do not impact any test cases.

* Use tuple of storage map instead of two parameters. This changes the
behavior of decl_storage genesis build closure (breaking api).

* Do not use build storage before assimilate.

* fix error

* Update core/state-machine/src/backend.rs
This commit is contained in:
cheme
2019-08-08 15:05:25 +02:00
committed by Gavin Wood
parent 0067b2d9a2
commit b0e1212d48
59 changed files with 523 additions and 271 deletions
+30 -6
View File
@@ -70,10 +70,14 @@ pub trait Backend<H: Hasher> {
/// Retrieve all entries keys of child storage and call `f` for each of those keys.
fn for_keys_in_child_storage<F: FnMut(&[u8])>(&self, storage_key: &[u8], f: F);
/// Retrieve all entries keys of which start with the given prefix and
/// Retrieve all entries keys which start with the given prefix and
/// call `f` for each of those keys.
fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F);
/// Retrieve all child entries keys which start with the given prefix and
/// call `f` for each of those keys.
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(&self, storage_key: &[u8], prefix: &[u8], f: F);
/// Calculate the storage root, with given delta over what is already stored in
/// the backend, and produce a "transaction" that can be used to commit.
/// Does not include child storage updates.
@@ -103,11 +107,7 @@ pub trait Backend<H: Hasher> {
/// Get all keys of child storage with given prefix
fn child_keys(&self, child_storage_key: &[u8], prefix: &[u8]) -> Vec<Vec<u8>> {
let mut all = Vec::new();
self.for_keys_in_child_storage(child_storage_key, |k| {
if k.starts_with(prefix) {
all.push(k.to_vec());
}
});
self.for_child_keys_with_prefix(child_storage_key, prefix, |k| all.push(k.to_vec()));
all
}
@@ -248,6 +248,25 @@ impl<H: Hasher> From<HashMap<Option<Vec<u8>>, HashMap<Vec<u8>, Vec<u8>>>> for In
}
}
impl<H: Hasher> From<(
HashMap<Vec<u8>, Vec<u8>>,
HashMap<Vec<u8>, HashMap<Vec<u8>, Vec<u8>>>,
)> for InMemory<H> {
fn from(inners: (
HashMap<Vec<u8>, Vec<u8>>,
HashMap<Vec<u8>, HashMap<Vec<u8>, Vec<u8>>>,
)) -> Self {
let mut inner: HashMap<Option<Vec<u8>>, HashMap<Vec<u8>, Vec<u8>>>
= inners.1.into_iter().map(|(k, v)| (Some(k), v)).collect();
inner.insert(None, inners.0);
InMemory {
inner: inner,
trie: None,
_hasher: PhantomData,
}
}
}
impl<H: Hasher> From<HashMap<Vec<u8>, Vec<u8>>> for InMemory<H> {
fn from(inner: HashMap<Vec<u8>, Vec<u8>>) -> Self {
let mut expanded = HashMap::new();
@@ -306,6 +325,11 @@ impl<H: Hasher> Backend<H> for InMemory<H> {
self.inner.get(&Some(storage_key.to_vec())).map(|map| map.keys().for_each(|k| f(&k)));
}
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(&self, storage_key: &[u8], prefix: &[u8], f: F) {
self.inner.get(&Some(storage_key.to_vec()))
.map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f));
}
fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
+37 -11
View File
@@ -20,7 +20,7 @@ use std::collections::HashMap;
use std::iter::FromIterator;
use crate::backend::{Backend, InMemory};
use hash_db::Hasher;
use trie::TrieConfiguration;
use trie::{TrieConfiguration, default_child_trie_root};
use trie::trie_types::Layout;
use primitives::offchain;
use primitives::storage::well_known_keys::is_child_storage_key;
@@ -35,13 +35,9 @@ pub struct BasicExternalities {
}
impl BasicExternalities {
/// Create a new instance of `BasicExternalities`
pub fn new(top: HashMap<Vec<u8>, Vec<u8>>) -> Self {
Self::new_with_children(top, Default::default())
}
/// Create a new instance of `BasicExternalities` with children
pub fn new_with_children(
/// Create a new instance of `BasicExternalities`
pub fn new(
top: HashMap<Vec<u8>, Vec<u8>>,
children: HashMap<Vec<u8>, HashMap<Vec<u8>, Vec<u8>>>,
) -> Self {
@@ -80,7 +76,7 @@ impl FromIterator<(Vec<u8>, Vec<u8>)> for BasicExternalities {
}
impl Default for BasicExternalities {
fn default() -> Self { Self::new(Default::default()) }
fn default() -> Self { Self::new(Default::default(), Default::default()) }
}
impl From<HashMap<Vec<u8>, Vec<u8>>> for BasicExternalities {
@@ -105,6 +101,10 @@ impl<H: Hasher> Externalities<H> for BasicExternalities where H::Out: Ord {
self.children.get(storage_key.as_ref()).and_then(|child| child.get(key)).cloned()
}
fn original_child_storage(&self, storage_key: ChildStorageKey<H>, key: &[u8]) -> Option<Vec<u8>> {
Externalities::<H>::child_storage(self, storage_key, key)
}
fn place_storage(&mut self, key: Vec<u8>, maybe_value: Option<Vec<u8>>) {
if is_child_storage_key(&key) {
warn!(target: "trie", "Refuse to set child storage key via main storage");
@@ -147,9 +147,32 @@ impl<H: Hasher> Externalities<H> for BasicExternalities where H::Out: Ord {
self.top.retain(|key, _| !key.starts_with(prefix));
}
fn clear_child_prefix(&mut self, storage_key: ChildStorageKey<H>, prefix: &[u8]) {
if let Some(child) = self.children.get_mut(storage_key.as_ref()) {
child.retain(|key, _| !key.starts_with(prefix));
}
}
fn chain_id(&self) -> u64 { 42 }
fn storage_root(&mut self) -> H::Out {
let mut top = self.top.clone();
let keys: Vec<_> = self.children.keys().map(|k| k.to_vec()).collect();
// Single child trie implementation currently allows using the same child
// empty root for all child trie. Using null storage key until multiple
// type of child trie support.
let empty_hash = default_child_trie_root::<Layout<H>>(&[]);
for storage_key in keys {
let child_root = self.child_storage_root(
ChildStorageKey::<H>::from_slice(storage_key.as_slice())
.expect("Map only feed by valid keys; qed")
);
if &empty_hash[..] == &child_root[..] {
top.remove(&storage_key);
} else {
top.insert(storage_key, child_root);
}
}
Layout::<H>::trie_root(self.top.clone())
}
@@ -159,7 +182,7 @@ impl<H: Hasher> Externalities<H> for BasicExternalities where H::Out: Ord {
InMemory::<H>::default().child_storage_root(storage_key.as_ref(), delta).0
} else {
vec![]
default_child_trie_root::<Layout<H>>(storage_key.as_ref())
}
}
@@ -212,7 +235,7 @@ mod tests {
fn children_works() {
let child_storage = b":child_storage:default:test".to_vec();
let mut ext = BasicExternalities::new_with_children(
let mut ext = BasicExternalities::new(
Default::default(),
map![
child_storage.clone() => map![
@@ -240,7 +263,10 @@ mod tests {
#[test]
fn basic_externalities_is_empty() {
// Make sure no values are set by default in `BasicExternalities`.
let (storage, child_storage) = BasicExternalities::new(Default::default()).into_storages();
let (storage, child_storage) = BasicExternalities::new(
Default::default(),
Default::default(),
).into_storages();
assert!(storage.is_empty());
assert!(child_storage.is_empty());
}
+26 -1
View File
@@ -204,6 +204,22 @@ where
self.backend.child_storage(storage_key.as_ref(), key).expect(EXT_NOT_ALLOWED_TO_FAIL))
}
fn child_storage_hash(&self, storage_key: ChildStorageKey<H>, key: &[u8]) -> Option<H::Out> {
let _guard = panic_handler::AbortGuard::force_abort();
self.overlay.child_storage(storage_key.as_ref(), key).map(|x| x.map(|x| H::hash(x))).unwrap_or_else(||
self.backend.storage_hash(key).expect(EXT_NOT_ALLOWED_TO_FAIL))
}
fn original_child_storage(&self, storage_key: ChildStorageKey<H>, key: &[u8]) -> Option<Vec<u8>> {
let _guard = panic_handler::AbortGuard::force_abort();
self.backend.child_storage(storage_key.as_ref(), key).expect(EXT_NOT_ALLOWED_TO_FAIL)
}
fn original_child_storage_hash(&self, storage_key: ChildStorageKey<H>, key: &[u8]) -> Option<H::Out> {
let _guard = panic_handler::AbortGuard::force_abort();
self.backend.child_storage_hash(storage_key.as_ref(), key).expect(EXT_NOT_ALLOWED_TO_FAIL)
}
fn exists_storage(&self, key: &[u8]) -> bool {
let _guard = panic_handler::AbortGuard::force_abort();
match self.overlay.storage(key) {
@@ -263,6 +279,16 @@ where
});
}
fn clear_child_prefix(&mut self, storage_key: ChildStorageKey<H>, prefix: &[u8]) {
let _guard = panic_handler::AbortGuard::force_abort();
self.mark_dirty();
self.overlay.clear_child_prefix(storage_key.as_ref(), prefix);
self.backend.for_child_keys_with_prefix(storage_key.as_ref(), prefix, |key| {
self.overlay.set_child_storage(storage_key.as_ref().to_vec(), key.to_vec(), None);
});
}
fn chain_id(&self) -> u64 {
42
}
@@ -276,7 +302,6 @@ where
let child_storage_keys =
self.overlay.prospective.children.keys()
.chain(self.overlay.committed.children.keys());
let child_delta_iter = child_storage_keys.map(|storage_key|
(storage_key.clone(), self.overlay.committed.children.get(storage_key)
.into_iter()
+21
View File
@@ -155,15 +155,33 @@ pub trait Externalities<H: Hasher> {
self.storage(key).map(|v| H::hash(&v))
}
/// Get child storage value hash. This may be optimized for large values.
fn child_storage_hash(&self, storage_key: ChildStorageKey<H>, key: &[u8]) -> Option<H::Out> {
self.child_storage(storage_key, key).map(|v| H::hash(&v))
}
/// Read original runtime storage, ignoring any overlayed changes.
fn original_storage(&self, key: &[u8]) -> Option<Vec<u8>>;
/// Read original runtime child storage, ignoring any overlayed changes.
fn original_child_storage(&self, storage_key: ChildStorageKey<H>, key: &[u8]) -> Option<Vec<u8>>;
/// Get original storage value hash, ignoring any overlayed changes.
/// This may be optimized for large values.
fn original_storage_hash(&self, key: &[u8]) -> Option<H::Out> {
self.original_storage(key).map(|v| H::hash(&v))
}
/// Get original child storage value hash, ignoring any overlayed changes.
/// This may be optimized for large values.
fn original_child_storage_hash(
&self,
storage_key: ChildStorageKey<H>,
key: &[u8],
) -> Option<H::Out> {
self.original_child_storage(storage_key, key).map(|v| H::hash(&v))
}
/// Read child runtime storage.
fn child_storage(&self, storage_key: ChildStorageKey<H>, key: &[u8]) -> Option<Vec<u8>>;
@@ -203,6 +221,9 @@ pub trait Externalities<H: Hasher> {
/// Clear storage entries which keys are start with the given prefix.
fn clear_prefix(&mut self, prefix: &[u8]);
/// Clear child storage entries which keys are start with the given prefix.
fn clear_child_prefix(&mut self, storage_key: ChildStorageKey<H>, prefix: &[u8]);
/// Set or clear a storage entry (`key`) of current contract being called (effective immediately).
fn place_storage(&mut self, key: Vec<u8>, value: Option<Vec<u8>>);
@@ -220,6 +220,38 @@ impl OverlayedChanges {
}
}
pub(crate) fn clear_child_prefix(&mut self, storage_key: &[u8], prefix: &[u8]) {
let extrinsic_index = self.extrinsic_index();
let map_entry = self.prospective.children.entry(storage_key.to_vec()).or_default();
for (key, entry) in map_entry.1.iter_mut() {
if key.starts_with(prefix) {
*entry = None;
if let Some(extrinsic) = extrinsic_index {
map_entry.0.get_or_insert_with(Default::default)
.insert(extrinsic);
}
}
}
if let Some(child_committed) = self.committed.children.get(storage_key) {
// Then do the same with keys from commited changes.
// NOTE that we are making changes in the prospective change set.
for key in child_committed.1.keys() {
if key.starts_with(prefix) {
let entry = map_entry.1.entry(key.clone()).or_default();
*entry = None;
if let Some(extrinsic) = extrinsic_index {
map_entry.0.get_or_insert_with(Default::default)
.insert(extrinsic);
}
}
}
}
}
/// Discard prospective changes to state.
pub fn discard_prospective(&mut self) {
self.prospective.clear();
@@ -174,6 +174,10 @@ impl<'a, S, H> Backend<H> for ProvingBackend<'a, S, H>
self.backend.for_keys_with_prefix(prefix, f)
}
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(&self, storage_key: &[u8], prefix: &[u8], f: F) {
self.backend.for_child_keys_with_prefix(storage_key, prefix, f)
}
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
self.backend.pairs()
}
+43 -30
View File
@@ -17,7 +17,6 @@
//! Test implementation for Externalities.
use std::collections::{HashMap};
use std::iter::FromIterator;
use hash_db::Hasher;
use crate::backend::{InMemory, Backend};
use primitives::storage::well_known_keys::is_child_storage_key;
@@ -46,22 +45,12 @@ pub struct TestExternalities<H: Hasher, N: ChangesTrieBlockNumber> {
impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N> {
/// Create a new instance of `TestExternalities` with storage.
pub fn new(storage: HashMap<Vec<u8>, Vec<u8>>) -> Self {
Self::new_with_children((storage, Default::default()))
}
/// Create a new instance of `TestExternalities` with storage and children.
pub fn new_with_children(storage: StorageTuple) -> Self {
Self::new_with_code_with_children(&[], storage)
pub fn new(storage: StorageTuple) -> Self {
Self::new_with_code(&[], storage)
}
/// Create a new instance of `TestExternalities` with code and storage.
pub fn new_with_code(code: &[u8], storage: HashMap<Vec<u8>, Vec<u8>>) -> Self {
Self::new_with_code_with_children(code, (storage, Default::default()))
}
/// Create a new instance of `TestExternalities` with code, storage and children.
pub fn new_with_code_with_children(code: &[u8], mut storage: StorageTuple) -> Self {
pub fn new_with_code(code: &[u8], mut storage: StorageTuple) -> Self {
let mut overlay = OverlayedChanges::default();
assert!(storage.0.keys().all(|key| !is_child_storage_key(key)));
@@ -137,25 +126,13 @@ impl<H: Hasher, N: ChangesTrieBlockNumber> PartialEq for TestExternalities<H, N>
}
}
impl<H: Hasher, N: ChangesTrieBlockNumber> FromIterator<(Vec<u8>, Vec<u8>)> for TestExternalities<H, N> {
fn from_iter<I: IntoIterator<Item=(Vec<u8>, Vec<u8>)>>(iter: I) -> Self {
Self::new(iter.into_iter().collect())
}
}
impl<H: Hasher, N: ChangesTrieBlockNumber> Default for TestExternalities<H, N> {
fn default() -> Self { Self::new(Default::default()) }
}
impl<H: Hasher, N: ChangesTrieBlockNumber> From<HashMap<Vec<u8>, Vec<u8>>> for TestExternalities<H, N> {
fn from(hashmap: HashMap<Vec<u8>, Vec<u8>>) -> Self {
Self::from_iter(hashmap)
}
}
impl<H: Hasher, N: ChangesTrieBlockNumber> From<StorageTuple> for TestExternalities<H, N> {
fn from(storage: StorageTuple) -> Self {
Self::new_with_children(storage)
Self::new(storage)
}
}
@@ -184,6 +161,13 @@ impl<H, N> Externalities<H> for TestExternalities<H, N>
)
}
fn original_child_storage(&self, storage_key: ChildStorageKey<H>, key: &[u8]) -> Option<Vec<u8>> {
self.backend
.child_storage(storage_key.as_ref(), key)
.map(|x| x.map(|x| x.to_vec()))
.expect(EXT_NOT_ALLOWED_TO_FAIL)
}
fn place_storage(&mut self, key: Vec<u8>, maybe_value: Option<Vec<u8>>) {
if is_child_storage_key(&key) {
panic!("Refuse to directly set child storage key");
@@ -225,20 +209,45 @@ impl<H, N> Externalities<H> for TestExternalities<H, N>
});
}
fn clear_child_prefix(&mut self, storage_key: ChildStorageKey<H>, prefix: &[u8]) {
self.overlay.clear_child_prefix(storage_key.as_ref(), prefix);
let backend = &self.backend;
let overlay = &mut self.overlay;
backend.for_child_keys_with_prefix(storage_key.as_ref(), prefix, |key| {
overlay.set_child_storage(storage_key.as_ref().to_vec(), key.to_vec(), None);
});
}
fn chain_id(&self) -> u64 { 42 }
fn storage_root(&mut self) -> H::Out {
let child_storage_keys =
self.overlay.prospective.children.keys()
.chain(self.overlay.committed.children.keys());
let child_delta_iter = child_storage_keys.map(|storage_key|
(storage_key.clone(), self.overlay.committed.children.get(storage_key)
.into_iter()
.flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.clone())))
.chain(self.overlay.prospective.children.get(storage_key)
.into_iter()
.flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.clone()))))));
// compute and memoize
let delta = self.overlay.committed.top.iter().map(|(k, v)| (k.clone(), v.value.clone()))
.chain(self.overlay.prospective.top.iter().map(|(k, v)| (k.clone(), v.value.clone())));
self.backend.full_storage_root(delta, child_delta_iter).0
self.backend.storage_root(delta).0
}
fn child_storage_root(&mut self, storage_key: ChildStorageKey<H>) -> Vec<u8> {
let storage_key = storage_key.as_ref();
let (root, _, _) = {
let (root, is_empty, _) = {
let delta = self.overlay.committed.children.get(storage_key)
.into_iter()
.flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.clone())))
@@ -248,7 +257,11 @@ impl<H, N> Externalities<H> for TestExternalities<H, N>
self.backend.child_storage_root(storage_key, delta)
};
self.overlay.set_storage(storage_key.into(), Some(root.clone()));
if is_empty {
self.overlay.set_storage(storage_key.into(), None);
} else {
self.overlay.set_storage(storage_key.into(), Some(root.clone()));
}
root
}
@@ -82,6 +82,10 @@ impl<S: TrieBackendStorage<H>, H: Hasher> Backend<H> for TrieBackend<S, H> where
self.essence.for_keys_in_child_storage(storage_key, f)
}
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(&self, storage_key: &[u8], prefix: &[u8], f: F) {
self.essence.for_child_keys_with_prefix(storage_key, prefix, f)
}
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
let mut read_overlay = S::Overlay::default();
let eph = Ephemeral::new(self.essence.backend_storage(), &mut read_overlay);
@@ -119,7 +119,27 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> {
}
/// Execute given closure for all keys starting with prefix.
pub fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], mut f: F) {
pub fn for_child_keys_with_prefix<F: FnMut(&[u8])>(&self, storage_key: &[u8], prefix: &[u8], f: F) {
let root_vec = match self.storage(storage_key) {
Ok(v) => v.unwrap_or(default_child_trie_root::<Layout<H>>(storage_key)),
Err(e) => {
debug!(target: "trie", "Error while iterating child storage: {}", e);
return;
}
};
let mut root = H::Out::default();
root.as_mut().copy_from_slice(&root_vec);
self.keys_with_prefix_inner(&root, prefix, f)
}
/// Execute given closure for all keys starting with prefix.
pub fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
self.keys_with_prefix_inner(&self.root, prefix, f)
}
fn keys_with_prefix_inner<F: FnMut(&[u8])>(&self, root: &H::Out, prefix: &[u8], mut f: F) {
let mut read_overlay = S::Overlay::default();
let eph = Ephemeral {
storage: &self.storage,
@@ -127,7 +147,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> {
};
let mut iter = move || -> Result<(), Box<TrieError<H::Out>>> {
let trie = TrieDB::<H>::new(&eph, &self.root)?;
let trie = TrieDB::<H>::new(&eph, root)?;
let mut iter = trie.iter()?;
iter.seek(prefix)?;
@@ -149,6 +169,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> {
debug!(target: "trie", "Error while iterating by prefix: {}", e);
}
}
}
pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> {