Remove requirement on Hash = H256, make Proposer return StorageChanges and Proof (#3860)

* Extend `Proposer` to optionally generate a proof of the proposal * Something * Refactor sr-api to not depend on client anymore * Fix benches * Apply suggestions from code review Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com> * Apply suggestions from code review * Introduce new `into_storage_changes` function * Switch to runtime api for `execute_block` and don't require `H256` anywhere in the code * Put the `StorageChanges` into the `Proposal` * Move the runtime api error to its own trait * Adds `StorageTransactionCache` to the runtime api This requires that we add `type NodeBlock = ` to the `impl_runtime_apis!` macro to work around some bugs in rustc :( * Remove `type NodeBlock` and switch to a "better" hack * Start using the transaction cache from the runtime api * Make it compile * Move `InMemory` to its own file * Make all tests work again * Return block, storage_changes and proof from Blockbuilder::bake() * Make sure that we use/set `storage_changes` when possible * Add test * Fix deadlock * Remove accidentally added folders * Introduce `RecordProof` as argument type to be more explicit * Update client/src/client.rs Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com> * Update primitives/state-machine/src/ext.rs Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com> * Integrates review feedback * Remove `unsafe` usage * Update client/block-builder/src/lib.rs Co-Authored-By: Benjamin Kampmann <ben@gnunicorn.org> * Update client/src/call_executor.rs * Bump versions Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com> Co-authored-by: Benjamin Kampmann <ben.kampmann@googlemail.com>
2026-04-30 11:57:56 +00:00 · 2020-01-10 10:48:32 +01:00
parent 74d6e660c6
commit fd6b29dd2c
140 changed files with 4860 additions and 3339 deletions
@@ -16,14 +16,23 @@

 //! The overlayed changes to state.

+use crate::{
+	backend::Backend, ChangesTrieTransaction,
+	changes_trie::{
+		NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig, BlockNumber, build_changes_trie,
+		Storage as ChangesTrieStorage,
+	},
+};
+
 #[cfg(test)]
 use std::iter::FromIterator;
 use std::collections::{HashMap, BTreeMap, BTreeSet};
-use codec::Decode;
-use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig};
+use codec::{Decode, Encode};
 use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, OwnedChildInfo, ChildInfo};
 use std::{mem, ops};

+use hash_db::Hasher;
+
 /// The overlayed changes to state to be queried on top of the backend.
 ///
 /// A transaction shares all prospective changes within an inner overlay
@@ -60,6 +69,92 @@ pub struct OverlayedChangeSet {
 	pub children: HashMap<Vec<u8>, (BTreeMap<Vec<u8>, OverlayedValue>, OwnedChildInfo)>,
 }

+/// A storage changes structure that can be generated by the data collected in [`OverlayedChanges`].
+///
+/// This contains all the changes to the storage and transactions to apply theses changes to the
+/// backend.
+pub struct StorageChanges<Transaction, H: Hasher, N: BlockNumber> {
+	/// All changes to the main storage.
+	///
+	/// A value of `None` means that it was deleted.
+	pub main_storage_changes: Vec<(Vec<u8>, Option<Vec<u8>>)>,
+	/// All changes to the child storages.
+	pub child_storage_changes: Vec<(Vec<u8>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>,
+	/// A transaction for the backend that contains all changes from
+	/// [`main_storage_changes`](Self::main_storage_changes) and from
+	/// [`child_storage_changes`](Self::child_storage_changes).
+	pub transaction: Transaction,
+	/// The storage root after applying the transaction.
+	pub transaction_storage_root: H::Out,
+	/// Contains the transaction for the backend for the changes trie.
+	///
+	/// If changes trie is disabled the value is set to `None`.
+	pub changes_trie_transaction: Option<ChangesTrieTransaction<H, N>>,
+}
+
+impl<Transaction, H: Hasher, N: BlockNumber> StorageChanges<Transaction, H, N> {
+	/// Deconstruct into the inner values
+	pub fn into_inner(self) -> (
+		Vec<(Vec<u8>, Option<Vec<u8>>)>,
+		Vec<(Vec<u8>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>,
+		Transaction,
+		H::Out,
+		Option<ChangesTrieTransaction<H, N>>,
+	) {
+		(
+			self.main_storage_changes,
+			self.child_storage_changes,
+			self.transaction,
+			self.transaction_storage_root,
+			self.changes_trie_transaction,
+		)
+	}
+}
+
+/// The storage transaction are calculated as part of the `storage_root` and
+/// `changes_trie_storage_root`. These transactions can be reused for importing the block into the
+/// storage. So, we cache them to not require a recomputation of those transactions.
+pub struct StorageTransactionCache<Transaction, H: Hasher, N: BlockNumber> {
+	/// Contains the changes for the main and the child storages as one transaction.
+	pub(crate) transaction: Option<Transaction>,
+	/// The storage root after applying the transaction.
+	pub(crate) transaction_storage_root: Option<H::Out>,
+	/// Contains the changes trie transaction.
+	pub(crate) changes_trie_transaction: Option<Option<ChangesTrieTransaction<H, N>>>,
+	/// The storage root after applying the changes trie transaction.
+	pub(crate) changes_trie_transaction_storage_root: Option<Option<H::Out>>,
+}
+
+impl<Transaction, H: Hasher, N: BlockNumber> StorageTransactionCache<Transaction, H, N> {
+	/// Reset the cached transactions.
+	pub fn reset(&mut self) {
+		*self = Self::default();
+	}
+}
+
+impl<Transaction, H: Hasher, N: BlockNumber> Default for StorageTransactionCache<Transaction, H, N> {
+	fn default() -> Self {
+		Self {
+			transaction: None,
+			transaction_storage_root: None,
+			changes_trie_transaction: None,
+			changes_trie_transaction_storage_root: None,
+		}
+	}
+}
+
+impl<Transaction: Default, H: Hasher, N: BlockNumber> Default for StorageChanges<Transaction, H, N> {
+	fn default() -> Self {
+		Self {
+			main_storage_changes: Default::default(),
+			child_storage_changes: Default::default(),
+			transaction: Default::default(),
+			transaction_storage_root: Default::default(),
+			changes_trie_transaction: None,
+		}
+	}
+}
+
 #[cfg(test)]
 impl FromIterator<(Vec<u8>, OverlayedValue)> for OverlayedChangeSet {
 	fn from_iter<T: IntoIterator<Item = (Vec<u8>, OverlayedValue)>>(iter: T) -> Self {
@@ -105,7 +200,7 @@ impl OverlayedChanges {
 		true
 	}

-	/// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered
+	/// Returns a double-Option: None if the key is unknown (i.e. and the query should be referred
 	/// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose
 	/// value has been set.
 	pub fn storage(&self, key: &[u8]) -> Option<Option<&[u8]>> {
@@ -114,7 +209,7 @@ impl OverlayedChanges {
 			.map(|x| x.value.as_ref().map(AsRef::as_ref))
 	}

-	/// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered
+	/// Returns a double-Option: None if the key is unknown (i.e. and the query should be referred
 	/// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose
 	/// value has been set.
 	pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option<Option<&[u8]>> {
@@ -237,7 +332,7 @@ impl OverlayedChanges {
 			}
 		}

-		// Then do the same with keys from commited changes.
+		// Then do the same with keys from committed changes.
 		// NOTE that we are making changes in the prospective change set.
 		for key in self.committed.top.keys() {
 			if key.starts_with(prefix) {
@@ -338,15 +433,61 @@ impl OverlayedChanges {
 		impl Iterator<Item=(Vec<u8>, (impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)>, OwnedChildInfo))>,
 	){
 		assert!(self.prospective.is_empty());
-		(self.committed.top.into_iter().map(|(k, v)| (k, v.value)),
+		(
+			self.committed.top.into_iter().map(|(k, v)| (k, v.value)),
 			self.committed.children.into_iter()
-				.map(|(sk, (v, ci))| (sk, (v.into_iter().map(|(k, v)| (k, v.value)), ci))))
+				.map(|(sk, (v, ci))| (sk, (v.into_iter().map(|(k, v)| (k, v.value)), ci)))
+		)
+	}
+
+	/// Convert this instance with all changes into a [`StorageChanges`] instance.
+	pub fn into_storage_changes<
+		B: Backend<H>, H: Hasher, N: BlockNumber, T: ChangesTrieStorage<H, N>
+	>(
+		self,
+		backend: &B,
+		changes_trie_storage: Option<&T>,
+		parent_hash: H::Out,
+		mut cache: StorageTransactionCache<B::Transaction, H, N>,
+	) -> Result<StorageChanges<B::Transaction, H, N>, String> where H::Out: Ord + Encode + 'static {
+		// If the transaction does not exist, we generate it.
+		if cache.transaction.is_none() {
+			self.storage_root(backend, &mut cache);
+		}
+
+		let (transaction, transaction_storage_root) = cache.transaction.take()
+			.and_then(|t| cache.transaction_storage_root.take().map(|tr| (t, tr)))
+			.expect("Transaction was be generated as part of `storage_root`; qed");
+
+		// If the transaction does not exist, we generate it.
+		if cache.changes_trie_transaction.is_none() {
+			self.changes_trie_root(
+				backend,
+				changes_trie_storage,
+				parent_hash,
+				false,
+				&mut cache,
+			).map_err(|_| "Failed to generate changes trie transaction")?;
+		}
+
+		let changes_trie_transaction = cache.changes_trie_transaction
+			.take()
+			.expect("Changes trie transaction was generated by `changes_trie_root`; qed");
+
+		let (main_storage_changes, child_storage_changes) = self.into_committed();
+
+		Ok(StorageChanges {
+			main_storage_changes: main_storage_changes.collect(),
+			child_storage_changes: child_storage_changes.map(|(sk, it)| (sk, it.0.collect())).collect(),
+			transaction,
+			transaction_storage_root,
+			changes_trie_transaction,
+		})
 	}

 	/// Inserts storage entry responsible for current extrinsic index.
 	#[cfg(test)]
 	pub(crate) fn set_extrinsic_index(&mut self, extrinsic_index: u32) {
-		use codec::Encode;
 		self.prospective.top.insert(EXTRINSIC_INDEX.to_vec(), OverlayedValue {
 			value: Some(extrinsic_index.encode()),
 			extrinsics: None,
@@ -356,7 +497,7 @@ impl OverlayedChanges {
 	/// Returns current extrinsic index to use in changes trie construction.
 	/// None is returned if it is not set or changes trie config is not set.
 	/// Persistent value (from the backend) can be ignored because runtime must
-	/// set this index before first and unset after last extrinsic is executied.
+	/// set this index before first and unset after last extrinsic is executed.
 	/// Changes that are made outside of extrinsics, are marked with
 	/// `NO_EXTRINSIC_INDEX` index.
 	fn extrinsic_index(&self) -> Option<u32> {
@@ -369,6 +510,75 @@ impl OverlayedChanges {
 		}
 	}

+	/// Generate the storage root using `backend` and all changes from `prospective` and `committed`.
+	///
+	/// Returns the storage root and caches storage transaction in the given `cache`.
+	pub fn storage_root<H: Hasher, N: BlockNumber, B: Backend<H>>(
+		&self,
+		backend: &B,
+		cache: &mut StorageTransactionCache<B::Transaction, H, N>,
+	) -> H::Out
+		where H::Out: Ord + Encode,
+	{
+		let child_storage_keys = self.prospective.children.keys()
+				.chain(self.committed.children.keys());
+		let child_delta_iter = child_storage_keys.map(|storage_key|
+			(
+				storage_key.clone(),
+				self.committed.children.get(storage_key)
+					.into_iter()
+					.flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))
+					.chain(
+						self.prospective.children.get(storage_key)
+							.into_iter()
+							.flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))
+					),
+				self.child_info(storage_key).cloned()
+					.expect("child info initialized in either committed or prospective"),
+			)
+		);
+
+		// compute and memoize
+		let delta = self.committed.top.iter().map(|(k, v)| (k.clone(), v.value.clone()))
+			.chain(self.prospective.top.iter().map(|(k, v)| (k.clone(), v.value.clone())));
+
+		let (root, transaction) = backend.full_storage_root(delta, child_delta_iter);
+
+		cache.transaction = Some(transaction);
+		cache.transaction_storage_root = Some(root);
+
+		root
+	}
+
+	/// Generate the changes trie root.
+	///
+	/// Returns the changes trie root and caches the storage transaction into the given `cache`.
+	///
+	/// # Panics
+	///
+	/// Panics on storage error, when `panic_on_storage_error` is set.
+	pub fn changes_trie_root<H: Hasher, N: BlockNumber, B: Backend<H>, T: ChangesTrieStorage<H, N>>(
+		&self,
+		backend: &B,
+		changes_trie_storage: Option<&T>,
+		parent_hash: H::Out,
+		panic_on_storage_error: bool,
+		cache: &mut StorageTransactionCache<B::Transaction, H, N>,
+	) -> Result<Option<H::Out>, ()> where H::Out: Ord + Encode + 'static {
+		build_changes_trie::<_, T, H, N>(
+			backend,
+			changes_trie_storage,
+			self,
+			parent_hash,
+			panic_on_storage_error,
+		).map(|r| {
+			let root = r.as_ref().map(|r| r.1).clone();
+			cache.changes_trie_transaction = Some(r.map(|(db, _, cache)| (db, cache)));
+			cache.changes_trie_transaction_storage_root = Some(root);
+			root
+		})
+	}
+
 	/// Get child info for a storage key.
 	/// Take the latest value so prospective first.
 	pub fn child_info(&self, storage_key: &[u8]) -> Option<&OwnedChildInfo> {
@@ -445,7 +655,7 @@ mod tests {
 	use sp_core::{
 		Blake2Hasher, traits::Externalities, storage::well_known_keys::EXTRINSIC_INDEX,
 	};
-	use crate::backend::InMemory;
+	use crate::InMemoryBackend;
 	use crate::changes_trie::InMemoryStorage as InMemoryChangesTrieStorage;
 	use crate::ext::Ext;
 	use super::*;
@@ -494,7 +704,7 @@ mod tests {
 			(b"dogglesworth".to_vec(), b"catXXX".to_vec()),
 			(b"doug".to_vec(), b"notadog".to_vec()),
 		].into_iter().collect();
-		let backend = InMemory::<Blake2Hasher>::from(initial);
+		let backend = InMemoryBackend::<Blake2Hasher>::from(initial);
 		let mut overlay = OverlayedChanges {
 			committed: vec![
 				(b"dog".to_vec(), Some(b"puppy".to_vec()).into()),
@@ -509,8 +719,10 @@ mod tests {
 		};

 		let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
+		let mut cache = StorageTransactionCache::default();
 		let mut ext = Ext::new(
 			&mut overlay,
+			&mut cache,
 			&backend,
 			Some(&changes_trie_storage),
 			None,