Remove requirement on Hash = H256, make Proposer return StorageChanges and Proof (#3860)

* Extend `Proposer` to optionally generate a proof of the proposal

* Something

* Refactor sr-api to not depend on client anymore

* Fix benches

* Apply suggestions from code review

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Apply suggestions from code review

* Introduce new `into_storage_changes` function

* Switch to runtime api for `execute_block` and don't require `H256`
anywhere in the code

* Put the `StorageChanges` into the `Proposal`

* Move the runtime api error to its own trait

* Adds `StorageTransactionCache` to the runtime api

This requires that we add `type NodeBlock = ` to the
`impl_runtime_apis!` macro to work around some bugs in rustc :(

* Remove `type NodeBlock` and switch to a "better" hack

* Start using the transaction cache from the runtime api

* Make it compile

* Move `InMemory` to its own file

* Make all tests work again

* Return block, storage_changes and proof from Blockbuilder::bake()

* Make sure that we use/set `storage_changes` when possible

* Add test

* Fix deadlock

* Remove accidentally added folders

* Introduce `RecordProof` as argument type to be more explicit

* Update client/src/client.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update primitives/state-machine/src/ext.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Integrates review feedback

* Remove `unsafe` usage

* Update client/block-builder/src/lib.rs

Co-Authored-By: Benjamin Kampmann <ben@gnunicorn.org>

* Update client/src/call_executor.rs

* Bump versions

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>
Co-authored-by: Benjamin Kampmann <ben.kampmann@googlemail.com>

substrate/primitives/state-machine/src/ext.rs

@@ -17,18 +17,16 @@
 //! Concrete externalities implementation.
 
 use crate::{
-	backend::Backend, OverlayedChanges,
-	changes_trie::{
-		Storage as ChangesTrieStorage, CacheAction as ChangesTrieCacheAction, build_changes_trie,
-	},
+	backend::Backend, OverlayedChanges, StorageTransactionCache,
+	changes_trie::Storage as ChangesTrieStorage,
 };
 
 use hash_db::Hasher;
 use sp_core::{
 	storage::{ChildStorageKey, well_known_keys::is_child_storage_key, ChildInfo},
-	traits::Externalities, hexdisplay::HexDisplay, hash::H256,
+	traits::Externalities, hexdisplay::HexDisplay,
 };
-use sp_trie::{trie_types::Layout, MemoryDB, default_child_trie_root};
+use sp_trie::{trie_types::Layout, default_child_trie_root};
 use sp_externalities::Extensions;
 use codec::{Decode, Encode};
 
@@ -67,23 +65,20 @@ impl<B: error::Error, E: error::Error> error::Error for Error<B, E> {
 }
 
 /// Wraps a read-only backend, call executor, and current overlayed changes.
-pub struct Ext<'a, H, N, B, T> where H: Hasher<Out=H256>, B: 'a + Backend<H> {
+pub struct Ext<'a, H, N, B, T>
+	where
+		H: Hasher,
+		B: 'a + Backend<H>,
+		N: crate::changes_trie::BlockNumber,
+{
 	/// The overlayed changes to write to.
 	overlay: &'a mut OverlayedChanges,
 	/// The storage backend to read from.
 	backend: &'a B,
-	/// The storage transaction necessary to commit to the backend. Is cached when
-	/// `storage_root` is called and the cache is cleared on every subsequent change.
-	storage_transaction: Option<(B::Transaction, H::Out)>,
+	/// The cache for the storage transactions.
+	storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H, N>,
 	/// Changes trie storage to read from.
 	changes_trie_storage: Option<&'a T>,
-	/// The changes trie transaction necessary to commit to the changes trie backend.
-	/// Set to Some when `storage_changes_root` is called. Could be replaced later
-	/// by calling `storage_changes_root` again => never used as cache.
-	/// This differs from `storage_transaction` behavior, because the moment when
-	/// `storage_changes_root` is called matters + we need to remember additional
-	/// data at this moment (block number).
-	changes_trie_transaction: Option<(MemoryDB<H>, H::Out, ChangesTrieCacheAction<H::Out, N>)>,
 	/// Pseudo-unique id used for tracing.
 	pub id: u16,
 	/// Dummy usage of N arg.
@@ -94,7 +89,8 @@ pub struct Ext<'a, H, N, B, T> where H: Hasher<Out=H256>, B: 'a + Backend<H> {
 
 impl<'a, H, N, B, T> Ext<'a, H, N, B, T>
 where
-	H: Hasher<Out=H256>,
+	H: Hasher,
+	H::Out: Ord + 'static + codec::Codec,
 	B: 'a + Backend<H>,
 	T: 'a + ChangesTrieStorage<H, N>,
 	N: crate::changes_trie::BlockNumber,
@@ -103,6 +99,7 @@ where
 	/// Create a new `Ext` from overlayed changes and read-only backend
 	pub fn new(
 		overlay: &'a mut OverlayedChanges,
+		storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H, N>,
 		backend: &'a B,
 		changes_trie_storage: Option<&'a T>,
 		extensions: Option<&'a mut Extensions>,
@@ -110,49 +107,27 @@ where
 		Ext {
 			overlay,
 			backend,
-			storage_transaction: None,
 			changes_trie_storage,
-			changes_trie_transaction: None,
+			storage_transaction_cache,
 			id: rand::random(),
 			_phantom: Default::default(),
 			extensions,
 		}
 	}
 
-	/// Get the transaction necessary to update the backend.
-	pub fn transaction(&mut self) -> (
-		(B::Transaction, H256),
-		Option<crate::ChangesTrieTransaction<H, N>>,
-	) {
-		let _ = self.storage_root();
-
-		let (storage_transaction, changes_trie_transaction) = (
-			self.storage_transaction
-				.take()
-				.expect("storage_transaction always set after calling storage root; qed"),
-			self.changes_trie_transaction
-				.take()
-				.map(|(tx, _, cache)| (tx, cache)),
-		);
-
-		(
-			storage_transaction,
-			changes_trie_transaction,
-		)
-	}
-
 	/// Invalidates the currently cached storage root and the db transaction.
 	///
 	/// Called when there are changes that likely will invalidate the storage root.
 	fn mark_dirty(&mut self) {
-		self.storage_transaction = None;
+		self.storage_transaction_cache.reset();
 	}
 }
 
 #[cfg(test)]
 impl<'a, H, N, B, T> Ext<'a, H, N, B, T>
 where
-	H: Hasher<Out=H256>,
+	H: Hasher,
+	H::Out: Ord + 'static,
 	B: 'a + Backend<H>,
 	T: 'a + ChangesTrieStorage<H, N>,
 	N: crate::changes_trie::BlockNumber,
@@ -173,7 +148,8 @@ where
 
 impl<'a, H, B, T, N> Externalities for Ext<'a, H, N, B, T>
 where
-	H: Hasher<Out=H256>,
+	H: Hasher,
+	H::Out: Ord + 'static + codec::Codec,
 	B: 'a + Backend<H>,
 	T: 'a + ChangesTrieStorage<H, N>,
 	N: crate::changes_trie::BlockNumber,
@@ -295,6 +271,7 @@ where
 			HexDisplay::from(&key),
 			result.as_ref().map(HexDisplay::from),
 		);
+
 		result
 	}
 
@@ -330,8 +307,8 @@ where
 			HexDisplay::from(&key),
 			result,
 		);
-		result
 
+		result
 	}
 
 	fn exists_child_storage(
@@ -499,7 +476,7 @@ where
 
 	fn storage_root(&mut self) -> Vec<u8> {
 		let _guard = sp_panic_handler::AbortGuard::force_abort();
-		if let Some((_, ref root)) = self.storage_transaction {
+		if let Some(ref root) = self.storage_transaction_cache.transaction_storage_root {
 			trace!(target: "state-trace", "{:04x}: Root (cached) {}",
 				self.id,
 				HexDisplay::from(&root.as_ref()),
@@ -507,35 +484,8 @@ where
 			return root.encode();
 		}
 
-		let child_storage_keys = self.overlay.prospective.children.keys()
-				.chain(self.overlay.committed.children.keys());
-		let child_delta_iter = child_storage_keys.map(|storage_key|
-			(
-				storage_key.clone(),
-				self.overlay.committed.children.get(storage_key)
-					.into_iter()
-					.flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))
-					.chain(
-						self.overlay.prospective.children.get(storage_key)
-							.into_iter()
-							.flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))
-					),
-				self.overlay.child_info(storage_key).cloned()
-					.expect("child info initialized in either committed or prospective"),
-			)
-		);
-
-
-		// compute and memoize
-		let delta = self.overlay.committed.top.iter().map(|(k, v)| (k.clone(), v.value.clone()))
-			.chain(self.overlay.prospective.top.iter().map(|(k, v)| (k.clone(), v.value.clone())));
-
-		let (root, transaction) = self.backend.full_storage_root(delta, child_delta_iter);
-		self.storage_transaction = Some((transaction, root));
-		trace!(target: "state-trace", "{:04x}: Root {}",
-			self.id,
-			HexDisplay::from(&root.as_ref()),
-		);
+		let root = self.overlay.storage_root(self.backend, self.storage_transaction_cache);
+		trace!(target: "state-trace", "{:04x}: Root {}", self.id, HexDisplay::from(&root.as_ref()));
 		root.encode()
 	}
 
@@ -544,7 +494,7 @@ where
 		storage_key: ChildStorageKey,
 	) -> Vec<u8> {
 		let _guard = sp_panic_handler::AbortGuard::force_abort();
-		if self.storage_transaction.is_some() {
+		if self.storage_transaction_cache.transaction_storage_root.is_some() {
 			let root = self
 				.storage(storage_key.as_ref())
 				.and_then(|k| Decode::decode(&mut &k[..]).ok())
@@ -612,35 +562,33 @@ where
 
 	fn storage_changes_root(&mut self, parent_hash: &[u8]) -> Result<Option<Vec<u8>>, ()> {
 		let _guard = sp_panic_handler::AbortGuard::force_abort();
-
-		self.changes_trie_transaction = build_changes_trie::<_, T, H, N>(
+		let root = self.overlay.changes_trie_root(
 			self.backend,
 			self.changes_trie_storage.clone(),
-			self.overlay,
-			H256::decode(&mut &parent_hash[..]).map_err(|e|
+			Decode::decode(&mut &parent_hash[..]).map_err(|e|
 				trace!(
 					target: "state-trace",
 					"Failed to decode changes root parent hash: {}",
 					e,
 				)
 			)?,
-		)?;
-		let result = Ok(
-			self.changes_trie_transaction.as_ref().map(|(_, root, _)| root.encode())
+			true,
+			self.storage_transaction_cache,
 		);
 
 		trace!(target: "state-trace", "{:04x}: ChangesRoot({}) {:?}",
 			self.id,
-			HexDisplay::from(&parent_hash.as_ref()),
-			result,
+			HexDisplay::from(&parent_hash),
+			root,
 		);
-		result
+
+		root.map(|r| r.map(|o| o.encode()))
 	}
 }
 
 impl<'a, H, B, T, N> sp_externalities::ExtensionStore for Ext<'a, H, N, B, T>
 where
-	H: Hasher<Out=H256>,
+	H: Hasher,
 	B: 'a + Backend<H>,
 	T: 'a + ChangesTrieStorage<H, N>,
 	N: crate::changes_trie::BlockNumber,
@@ -655,16 +603,16 @@ mod tests {
 	use super::*;
 	use hex_literal::hex;
 	use codec::Encode;
-	use sp_core::{Blake2Hasher, storage::well_known_keys::EXTRINSIC_INDEX, map};
+	use sp_core::{H256, Blake2Hasher, storage::well_known_keys::EXTRINSIC_INDEX, map};
 	use crate::{
 		changes_trie::{
 			Configuration as ChangesTrieConfiguration,
 			InMemoryStorage as InMemoryChangesTrieStorage,
-		}, backend::InMemory, overlayed_changes::OverlayedValue,
+		}, InMemoryBackend, overlayed_changes::OverlayedValue,
 	};
 	use sp_core::storage::{Storage, StorageChild};
 
-	type TestBackend = InMemory<Blake2Hasher>;
+	type TestBackend = InMemoryBackend<Blake2Hasher>;
 	type TestChangesTrieStorage = InMemoryChangesTrieStorage<Blake2Hasher, u64>;
 	type TestExt<'a> = Ext<'a, Blake2Hasher, u64, TestBackend, TestChangesTrieStorage>;
 
@@ -691,27 +639,30 @@ mod tests {
 	#[test]
 	fn storage_changes_root_is_none_when_storage_is_not_provided() {
 		let mut overlay = prepare_overlay_with_changes();
+		let mut cache = StorageTransactionCache::default();
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &backend, None, None);
+		let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 		assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None);
 	}
 
 	#[test]
 	fn storage_changes_root_is_none_when_extrinsic_changes_are_none() {
 		let mut overlay = prepare_overlay_with_changes();
+		let mut cache = StorageTransactionCache::default();
 		overlay.changes_trie_config = None;
 		let storage = TestChangesTrieStorage::with_blocks(vec![(100, Default::default())]);
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None);
+		let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, Some(&storage), None);
 		assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None);
 	}
 
 	#[test]
 	fn storage_changes_root_is_some_when_extrinsic_changes_are_non_empty() {
 		let mut overlay = prepare_overlay_with_changes();
+		let mut cache = StorageTransactionCache::default();
 		let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None);
+		let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, Some(&storage), None);
 		assert_eq!(
 			ext.storage_changes_root(&H256::default().encode()).unwrap(),
 			Some(hex!("bb0c2ef6e1d36d5490f9766cfcc7dfe2a6ca804504c3bb206053890d6dd02376").to_vec()),
@@ -721,10 +672,11 @@ mod tests {
 	#[test]
 	fn storage_changes_root_is_some_when_extrinsic_changes_are_empty() {
 		let mut overlay = prepare_overlay_with_changes();
+		let mut cache = StorageTransactionCache::default();
 		overlay.prospective.top.get_mut(&vec![1]).unwrap().value = None;
 		let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None);
+		let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, Some(&storage), None);
 		assert_eq!(
 			ext.storage_changes_root(&H256::default().encode()).unwrap(),
 			Some(hex!("96f5aae4690e7302737b6f9b7f8567d5bbb9eac1c315f80101235a92d9ec27f4").to_vec()),
@@ -733,6 +685,7 @@ mod tests {
 
 	#[test]
 	fn next_storage_key_works() {
+		let mut cache = StorageTransactionCache::default();
 		let mut overlay = OverlayedChanges::default();
 		overlay.set_storage(vec![20], None);
 		overlay.set_storage(vec![30], Some(vec![31]));
@@ -745,7 +698,7 @@ mod tests {
 			children: map![]
 		}.into();
 
-		let ext = TestExt::new(&mut overlay, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		// next_backend < next_overlay
 		assert_eq!(ext.next_storage_key(&[5]), Some(vec![10]));
@@ -761,7 +714,7 @@ mod tests {
 
 		drop(ext);
 		overlay.set_storage(vec![50], Some(vec![50]));
-		let ext = TestExt::new(&mut overlay, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		// next_overlay exist but next_backend doesn't exist
 		assert_eq!(ext.next_storage_key(&[40]), Some(vec![50]));
@@ -774,7 +727,7 @@ mod tests {
 		const CHILD_UUID_1: &[u8] = b"unique_id_1";
 		const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(CHILD_UUID_1);
 
-
+		let mut cache = StorageTransactionCache::default();
 		let child = || ChildStorageKey::from_slice(CHILD_KEY_1).unwrap();
 		let mut overlay = OverlayedChanges::default();
 		overlay.set_child_storage(child().as_ref().to_vec(), CHILD_INFO_1, vec![20], None);
@@ -794,7 +747,7 @@ mod tests {
 		}.into();
 
 
-		let ext = TestExt::new(&mut overlay, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		// next_backend < next_overlay
 		assert_eq!(ext.next_child_storage_key(child(), CHILD_INFO_1, &[5]), Some(vec![10]));
@@ -810,7 +763,7 @@ mod tests {
 
 		drop(ext);
 		overlay.set_child_storage(child().as_ref().to_vec(), CHILD_INFO_1, vec![50], Some(vec![50]));
-		let ext = TestExt::new(&mut overlay, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		// next_overlay exist but next_backend doesn't exist
 		assert_eq!(ext.next_child_storage_key(child(), CHILD_INFO_1, &[40]), Some(vec![50]));