Multiple storage root support (#902)

* Implement a non generic version of child delta trie * Use delta_trie_root in state_machine * Expand InMemory backend to support multi-storage * Create Consolidate trait * Fix all crate compile and remove unused OverlayedChanges::drain * Implement child storage root support and overlay changes * Add child storage reader * Add child storage writer * Implement child storage cleaning * Fix light backend compile * Add all required ext functions for wasm executor * Add ext def to io * Add all io functions * Fix nostd compile * Add simple test * Remove unnecessary vec copy in child_storage_root_transaction * Use values_mut/for_each to make it shorter * Use extend to shorter a for loop * Move record_all_keys to trie so it's easier to generic them later * space -> tab * Remove to_owned in debug format * Clean out all to_owned * Break debug_trace to multiple lines * Remove 0.. * UserError copy/paste typo * Replace Vec::from_raw_parts by slice::from_raw_parts * Use iter::empty() * Wrap some long lines * Wrap a missing line * Remove unnecessary map https://github.com/paritytech/substrate/pull/856#discussion_r226222663 * Call ext_free after from_raw_parts * Fix tests in other crates
2026-06-13 09:21:05 +00:00 · 2018-10-19 00:54:02 +08:00
parent 849506daa5
commit 2604474880
18 changed files with 1000 additions and 111 deletions
@@ -23,7 +23,7 @@ use std::marker::PhantomData;
 use hash_db::Hasher;
 use trie_backend::TrieBackend;
 use trie_backend_essence::TrieBackendStorage;
-use substrate_trie::{TrieDBMut, TrieMut, MemoryDB, trie_root};
+use substrate_trie::{TrieDBMut, TrieMut, MemoryDB, trie_root, child_trie_root};
 use heapsize::HeapSizeOf;

 /// A state backend is used to read state data and can have changes committed
@@ -35,7 +35,7 @@ pub trait Backend<H: Hasher> {
 	type Error: super::Error;

 	/// Storage changes to be applied if committing
-	type Transaction;
+	type Transaction: Consolidate + Default;

 	/// Type of trie backend storage.
 	type TrieBackendStorage: TrieBackendStorage<H>;
@@ -43,11 +43,22 @@ pub trait Backend<H: Hasher> {
 	/// Get keyed storage associated with specific address, or None if there is nothing associated.
 	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error>;

+	/// Get keyed child storage associated with specific address, or None if there is nothing associated.
+	fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error>;
+
 	/// true if a key exists in storage.
 	fn exists_storage(&self, key: &[u8]) -> Result<bool, Self::Error> {
 		Ok(self.storage(key)?.is_some())
 	}

+	/// true if a key exists in child storage.
+	fn exists_child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result<bool, Self::Error> {
+		Ok(self.child_storage(storage_key, key)?.is_some())
+	}
+
+	/// Retrieve all entries keys of child storage and call `f` for each of those keys.
+	fn for_keys_in_child_storage<F: FnMut(&[u8])>(&self, storage_key: &[u8], f: F);
+
 	/// Retrieve all entries keys of which start with the given prefix and
 	/// call `f` for each of those keys.
 	fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F);
@@ -59,6 +70,13 @@ pub trait Backend<H: Hasher> {
 		I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
 		H::Out: Ord;

+	/// Calculate the child storage root, with given delta over what is already stored in
+	/// the backend, and produce a "transaction" that can be used to commit.
+	fn child_storage_root<I>(&self, storage_key: &[u8], delta: I) -> (Vec<u8>, Self::Transaction)
+	where
+		I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
+		H::Out: Ord;
+
 	/// Get all key/value pairs into a Vec.
 	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)>;

@@ -66,6 +84,30 @@ pub trait Backend<H: Hasher> {
 	fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H>>;
 }

+/// Trait that allows consolidate two transactions together.
+pub trait Consolidate {
+	/// Consolidate two transactions into one.
+	fn consolidate(&mut self, other: Self);
+}
+
+impl Consolidate for () {
+	fn consolidate(&mut self, _: Self) {
+		()
+	}
+}
+
+impl Consolidate for Vec<(Option<Vec<u8>>, Vec<u8>, Option<Vec<u8>>)> {
+	fn consolidate(&mut self, mut other: Self) {
+		self.append(&mut other);
+	}
+}
+
+impl<H: Hasher> Consolidate for MemoryDB<H> {
+	fn consolidate(&mut self, other: Self) {
+		MemoryDB::consolidate(self, other)
+	}
+}
+
 /// Error impossible.
 // TODO: use `!` type when stabilized.
 #[derive(Debug)]
@@ -85,7 +127,7 @@ impl error::Error for Void {
 /// tests.
 #[derive(Eq)]
 pub struct InMemory<H> {
-	inner: HashMap<Vec<u8>, Vec<u8>>,
+	inner: HashMap<Option<Vec<u8>>, HashMap<Vec<u8>, Vec<u8>>>,
 	_hasher: PhantomData<H>,
 }

@@ -117,10 +159,10 @@ impl<H: Hasher> InMemory<H> where H::Out: HeapSizeOf {
 	/// Copy the state, with applied updates
 	pub fn update(&self, changes: <Self as Backend<H>>::Transaction) -> Self {
 		let mut inner: HashMap<_, _> = self.inner.clone();
-		for (key, val) in changes {
+		for (storage_key, key, val) in changes {
 			match val {
-				Some(v) => { inner.insert(key, v); },
-				None => { inner.remove(&key); },
+				Some(v) => { inner.entry(storage_key).or_default().insert(key, v); },
+				None => { inner.entry(storage_key).or_default().remove(&key); },
 			}
 		}

@@ -128,8 +170,8 @@ impl<H: Hasher> InMemory<H> where H::Out: HeapSizeOf {
 	}
 }

-impl<H> From<HashMap<Vec<u8>, Vec<u8>>> for InMemory<H> {
-	fn from(inner: HashMap<Vec<u8>, Vec<u8>>) -> Self {
+impl<H> From<HashMap<Option<Vec<u8>>, HashMap<Vec<u8>, Vec<u8>>>> for InMemory<H> {
+	fn from(inner: HashMap<Option<Vec<u8>>, HashMap<Vec<u8>, Vec<u8>>>) -> Self {
 		InMemory {
 			inner: inner,
 			_hasher: PhantomData,
@@ -137,23 +179,42 @@ impl<H> From<HashMap<Vec<u8>, Vec<u8>>> for InMemory<H> {
 	}
 }

+impl<H> From<HashMap<Vec<u8>, Vec<u8>>> for InMemory<H> {
+	fn from(inner: HashMap<Vec<u8>, Vec<u8>>) -> Self {
+		let mut expanded = HashMap::new();
+		expanded.insert(None, inner);
+		InMemory {
+			inner: expanded,
+			_hasher: PhantomData,
+		}
+	}
+}
+
 impl super::Error for Void {}

 impl<H: Hasher> Backend<H> for InMemory<H> where H::Out: HeapSizeOf {
 	type Error = Void;
-	type Transaction = Vec<(Vec<u8>, Option<Vec<u8>>)>;
+	type Transaction = Vec<(Option<Vec<u8>>, Vec<u8>, Option<Vec<u8>>)>;
 	type TrieBackendStorage = MemoryDB<H>;

 	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
-		Ok(self.inner.get(key).map(Clone::clone))
+		Ok(self.inner.get(&None).and_then(|map| map.get(key).map(Clone::clone)))
+	}
+
+	fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
+		Ok(self.inner.get(&Some(storage_key.to_vec())).and_then(|map| map.get(key).map(Clone::clone)))
 	}

 	fn exists_storage(&self, key: &[u8]) -> Result<bool, Self::Error> {
-		Ok(self.inner.get(key).is_some())
+		Ok(self.inner.get(&None).map(|map| map.get(key).is_some()).unwrap_or(false))
 	}

 	fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
-		self.inner.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f);
+		self.inner.get(&None).map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f));
+	}
+
+	fn for_keys_in_child_storage<F: FnMut(&[u8])>(&self, storage_key: &[u8], mut f: F) {
+		self.inner.get(&Some(storage_key.to_vec())).map(|map| map.keys().for_each(|k| f(&k)));
 	}

 	fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
@@ -161,7 +222,7 @@ impl<H: Hasher> Backend<H> for InMemory<H> where H::Out: HeapSizeOf {
 		I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
 		<H as Hasher>::Out: Ord,
 	{
-		let existing_pairs = self.inner.iter().map(|(k, v)| (k.clone(), Some(v.clone())));
+		let existing_pairs = self.inner.get(&None).into_iter().flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));

 		let transaction: Vec<_> = delta.into_iter().collect();
 		let root = trie_root::<H, _, _, _>(existing_pairs.chain(transaction.iter().cloned())
@@ -170,16 +231,52 @@ impl<H: Hasher> Backend<H> for InMemory<H> where H::Out: HeapSizeOf {
 			.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
 		);

-		(root, transaction)
+		let full_transaction = transaction.into_iter().map(|(k, v)| (None, k, v)).collect();
+
+		(root, full_transaction)
+	}
+
+	fn child_storage_root<I>(&self, storage_key: &[u8], delta: I) -> (Vec<u8>, Self::Transaction)
+	where
+		I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
+		H::Out: Ord
+	{
+		let storage_key = storage_key.to_vec();
+
+		let existing_pairs = self.inner.get(&Some(storage_key.clone())).into_iter().flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
+
+		let transaction: Vec<_> = delta.into_iter().collect();
+		let root = child_trie_root::<H, _, _, _>(
+			&storage_key,
+			existing_pairs.chain(transaction.iter().cloned())
+				.collect::<HashMap<_, _>>()
+				.into_iter()
+				.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
+		);
+
+		let full_transaction = transaction.into_iter().map(|(k, v)| (Some(storage_key.clone()), k, v)).collect();
+
+		(root, full_transaction)
 	}

 	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
-		self.inner.iter().map(|(k, v)| (k.clone(), v.clone())).collect()
+		self.inner.get(&None).into_iter().flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone()))).collect()
 	}

 	fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H>> {
 		let mut mdb = MemoryDB::default();	// TODO: should be more correct and use ::new()
-		let root = insert_into_memory_db::<H, _>(&mut mdb, self.inner.into_iter())?;
+		let mut root = None;
+		for (storage_key, map) in self.inner {
+			if storage_key != None {
+				let _ = insert_into_memory_db::<H, _>(&mut mdb, map.into_iter())?;
+			} else {
+				root = Some(insert_into_memory_db::<H, _>(&mut mdb, map.into_iter())?);
+			}
+		}
+		let root = match root {
+			Some(root) => root,
+			None => insert_into_memory_db::<H, _>(&mut mdb, ::std::iter::empty())?,
+		};
 		Some(TrieBackend::new(mdb, root))
 	}
 }