Remove requirement on Hash = H256, make Proposer return StorageChanges and Proof (#3860)

* Extend `Proposer` to optionally generate a proof of the proposal

* Something

* Refactor sr-api to not depend on client anymore

* Fix benches

* Apply suggestions from code review

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Apply suggestions from code review

* Introduce new `into_storage_changes` function

* Switch to runtime api for `execute_block` and don't require `H256`
anywhere in the code

* Put the `StorageChanges` into the `Proposal`

* Move the runtime api error to its own trait

* Adds `StorageTransactionCache` to the runtime api

This requires that we add `type NodeBlock = ` to the
`impl_runtime_apis!` macro to work around some bugs in rustc :(

* Remove `type NodeBlock` and switch to a "better" hack

* Start using the transaction cache from the runtime api

* Make it compile

* Move `InMemory` to its own file

* Make all tests work again

* Return block, storage_changes and proof from Blockbuilder::bake()

* Make sure that we use/set `storage_changes` when possible

* Add test

* Fix deadlock

* Remove accidentally added folders

* Introduce `RecordProof` as argument type to be more explicit

* Update client/src/client.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update primitives/state-machine/src/ext.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Integrates review feedback

* Remove `unsafe` usage

* Update client/block-builder/src/lib.rs

Co-Authored-By: Benjamin Kampmann <ben@gnunicorn.org>

* Update client/src/call_executor.rs

* Bump versions

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>
Co-authored-by: Benjamin Kampmann <ben.kampmann@googlemail.com>

substrate/client/api/src/backend.rs

@@ -21,8 +21,7 @@ use std::collections::HashMap;
 use sp_core::ChangesTrieConfiguration;
 use sp_core::offchain::OffchainStorage;
 use sp_runtime::{generic::BlockId, Justification, Storage};
-use sp_runtime::traits::{Block as BlockT, NumberFor};
-use sp_state_machine::backend::Backend as StateBackend;
+use sp_runtime::traits::{Block as BlockT, NumberFor, HasherFor};
 use sp_state_machine::{ChangesTrieStorage as StateChangesTrieStorage, ChangesTrieTransaction};
 use crate::{
 	blockchain::{
@@ -33,9 +32,19 @@ use crate::{
 };
 use sp_blockchain;
 use sp_consensus::BlockOrigin;
-use hash_db::Hasher;
 use parking_lot::RwLock;
 
+pub use sp_state_machine::Backend as StateBackend;
+
+/// Extracts the state backend type for the given backend.
+pub type StateBackendFor<B, Block> = <B as Backend<Block>>::State;
+
+/// Extracts the transaction for the given state backend.
+pub type TransactionForSB<B, Block> = <B as StateBackend<HasherFor<Block>>>::Transaction;
+
+/// Extracts the transaction for the given backend.
+pub type TransactionFor<B, Block> = TransactionForSB<StateBackendFor<B, Block>, Block>;
+
 /// In memory array of storage values.
 pub type StorageCollection = Vec<(Vec<u8>, Option<Vec<u8>>)>;
 
@@ -62,11 +71,7 @@ pub struct ImportSummary<Block: BlockT> {
 }
 
 /// Import operation wrapper
-pub struct ClientImportOperation<
-	Block: BlockT,
-	H: Hasher<Out=Block::Hash>,
-	B: Backend<Block, H>,
-> {
+pub struct ClientImportOperation<Block: BlockT, B: Backend<Block>> {
 	/// DB Operation.
 	pub op: B::BlockImportOperation,
 	/// Summary of imported block.
@@ -107,12 +112,9 @@ impl NewBlockState {
 /// Block insertion operation.
 ///
 /// Keeps hold if the inserted block state and data.
-pub trait BlockImportOperation<Block, H> where
-	Block: BlockT,
-	H: Hasher<Out=Block::Hash>,
-{
+pub trait BlockImportOperation<Block: BlockT> {
 	/// Associated state backend type.
-	type State: StateBackend<H>;
+	type State: StateBackend<HasherFor<Block>>;
 
 	/// Returns pending state.
 	///
@@ -132,10 +134,13 @@ pub trait BlockImportOperation<Block, H> where
 	fn update_cache(&mut self, cache: HashMap<well_known_cache_keys::Id, Vec<u8>>);
 
 	/// Inject storage data into the database.
-	fn update_db_storage(&mut self, update: <Self::State as StateBackend<H>>::Transaction) -> sp_blockchain::Result<()>;
+	fn update_db_storage(
+		&mut self,
+		update: TransactionForSB<Self::State, Block>,
+	) -> sp_blockchain::Result<()>;
 
 	/// Inject storage data into the database replacing any existing data.
-	fn reset_storage(&mut self, storage: Storage) -> sp_blockchain::Result<H::Out>;
+	fn reset_storage(&mut self, storage: Storage) -> sp_blockchain::Result<Block::Hash>;
 
 	/// Set storage changes.
 	fn update_storage(
@@ -145,7 +150,10 @@ pub trait BlockImportOperation<Block, H> where
 	) -> sp_blockchain::Result<()>;
 
 	/// Inject changes trie data into the database.
-	fn update_changes_trie(&mut self, update: ChangesTrieTransaction<H, NumberFor<Block>>) -> sp_blockchain::Result<()>;
+	fn update_changes_trie(
+		&mut self,
+		update: ChangesTrieTransaction<HasherFor<Block>, NumberFor<Block>>,
+	) -> sp_blockchain::Result<()>;
 
 	/// Insert auxiliary keys.
 	///
@@ -154,13 +162,18 @@ pub trait BlockImportOperation<Block, H> where
 		where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>;
 
 	/// Mark a block as finalized.
-	fn mark_finalized(&mut self, id: BlockId<Block>, justification: Option<Justification>) -> sp_blockchain::Result<()>;
-	/// Mark a block as new head. If both block import and set head are specified, set head overrides block import's best block rule.
+	fn mark_finalized(
+		&mut self,
+		id: BlockId<Block>,
+		justification: Option<Justification>,
+	) -> sp_blockchain::Result<()>;
+	/// Mark a block as new head. If both block import and set head are specified, set head
+	/// overrides block import's best block rule.
 	fn mark_head(&mut self, id: BlockId<Block>) -> sp_blockchain::Result<()>;
 }
 
 /// Finalize Facilities
-pub trait Finalizer<Block: BlockT, H: Hasher<Out=Block::Hash>, B: Backend<Block, H>> {
+pub trait Finalizer<Block: BlockT, B: Backend<Block>> {
 	/// Mark all blocks up to given as finalized in operation.
 	///
 	/// If `justification` is provided it is stored with the given finalized
@@ -172,7 +185,7 @@ pub trait Finalizer<Block: BlockT, H: Hasher<Out=Block::Hash>, B: Backend<Block,
 	/// best block should use `SelectChain` instead of the client.
 	fn apply_finality(
 		&self,
-		operation: &mut ClientImportOperation<Block, H, B>,
+		operation: &mut ClientImportOperation<Block, B>,
 		id: BlockId<Block>,
 		justification: Option<Justification>,
 		notify: bool,
@@ -226,20 +239,17 @@ pub trait AuxStore {
 /// should not be pruned. The backend should internally reference-count
 /// its state objects.
 ///
-/// The same applies for live `BlockImportOperation`s: while an import operation building on a parent `P`
-/// is alive, the state for `P` should not be pruned.
-pub trait Backend<Block, H>: AuxStore + Send + Sync where
-	Block: BlockT,
-	H: Hasher<Out=Block::Hash>,
-{
+/// The same applies for live `BlockImportOperation`s: while an import operation building on a
+/// parent `P` is alive, the state for `P` should not be pruned.
+pub trait Backend<Block: BlockT>: AuxStore + Send + Sync {
 	/// Associated block insertion operation type.
-	type BlockImportOperation: BlockImportOperation<Block, H, State=Self::State>;
+	type BlockImportOperation: BlockImportOperation<Block, State = Self::State>;
 	/// Associated blockchain backend type.
 	type Blockchain: BlockchainBackend<Block>;
 	/// Associated state backend type.
-	type State: StateBackend<H>;
+	type State: StateBackend<HasherFor<Block>> + Send;
 	/// Changes trie storage.
-	type ChangesTrieStorage: PrunableStateChangesTrieStorage<Block, H>;
+	type ChangesTrieStorage: PrunableStateChangesTrieStorage<Block>;
 	/// Offchain workers local storage.
 	type OffchainStorage: OffchainStorage;
 
@@ -249,7 +259,11 @@ pub trait Backend<Block, H>: AuxStore + Send + Sync where
 	fn begin_operation(&self) -> sp_blockchain::Result<Self::BlockImportOperation>;
 
 	/// Note an operation to contain state transition.
-	fn begin_state_operation(&self, operation: &mut Self::BlockImportOperation, block: BlockId<Block>) -> sp_blockchain::Result<()>;
+	fn begin_state_operation(
+		&self,
+		operation: &mut Self::BlockImportOperation,
+		block: BlockId<Block>,
+	) -> sp_blockchain::Result<()>;
 
 	/// Commit block insertion.
 	fn commit_operation(&self, transaction: Self::BlockImportOperation) -> sp_blockchain::Result<()>;
@@ -257,7 +271,11 @@ pub trait Backend<Block, H>: AuxStore + Send + Sync where
 	/// Finalize block with given Id.
 	///
 	/// This should only be called if the parent of the given block has been finalized.
-	fn finalize_block(&self, block: BlockId<Block>, justification: Option<Justification>) -> sp_blockchain::Result<()>;
+	fn finalize_block(
+		&self,
+		block: BlockId<Block>,
+		justification: Option<Justification>,
+	) -> sp_blockchain::Result<()>;
 
 	/// Returns reference to blockchain backend.
 	fn blockchain(&self) -> &Self::Blockchain;
@@ -321,8 +339,8 @@ pub trait Backend<Block, H>: AuxStore + Send + Sync where
 }
 
 /// Changes trie storage that supports pruning.
-pub trait PrunableStateChangesTrieStorage<Block: BlockT, H: Hasher>:
-	StateChangesTrieStorage<H, NumberFor<Block>>
+pub trait PrunableStateChangesTrieStorage<Block: BlockT>:
+	StateChangesTrieStorage<HasherFor<Block>, NumberFor<Block>>
 {
 	/// Get number block of oldest, non-pruned changes trie.
 	fn oldest_changes_trie_block(
@@ -333,18 +351,10 @@ pub trait PrunableStateChangesTrieStorage<Block: BlockT, H: Hasher>:
 }
 
 /// Mark for all Backend implementations, that are making use of state data, stored locally.
-pub trait LocalBackend<Block, H>: Backend<Block, H>
-where
-	Block: BlockT,
-	H: Hasher<Out=Block::Hash>,
-{}
+pub trait LocalBackend<Block: BlockT>: Backend<Block> {}
 
 /// Mark for all Backend implementations, that are fetching required state data from remote nodes.
-pub trait RemoteBackend<Block, H>: Backend<Block, H>
-where
-	Block: BlockT,
-	H: Hasher<Out=Block::Hash>,
-{
+pub trait RemoteBackend<Block: BlockT>: Backend<Block> {
 	/// Returns true if the state for given block is available locally.
 	fn is_local_state_available(&self, block: &BlockId<Block>) -> bool;
 

substrate/client/api/src/call_executor.rs

@@ -16,33 +16,28 @@
 
 //! A method call executor interface.
 
-use std::{cmp::Ord, panic::UnwindSafe, result, cell::RefCell};
+use std::{panic::UnwindSafe, result, cell::RefCell};
 use codec::{Encode, Decode};
 use sp_runtime::{
-	generic::BlockId, traits::Block as BlockT, traits::NumberFor,
+	generic::BlockId, traits::{Block as BlockT, HasherFor},
 };
 use sp_state_machine::{
-	self, OverlayedChanges, ExecutionManager, ExecutionStrategy,
-	ChangesTrieTransaction, StorageProof,
+	OverlayedChanges, ExecutionManager, ExecutionStrategy, StorageProof,
 };
 use sc_executor::{RuntimeVersion, NativeVersion};
 use sp_externalities::Extensions;
-use hash_db::Hasher;
-use sp_core::{Blake2Hasher, NativeOrEncoded};
+use sp_core::NativeOrEncoded;
 
-use sp_api::{ProofRecorder, InitializeBlock};
-use sp_blockchain;
+use sp_api::{ProofRecorder, InitializeBlock, StorageTransactionCache};
 
 /// Method call executor.
-pub trait CallExecutor<B, H>
-where
-	B: BlockT,
-	H: Hasher<Out=B::Hash>,
-	H::Out: Ord,
-{
+pub trait CallExecutor<B: BlockT> {
 	/// Externalities error type.
 	type Error: sp_state_machine::Error;
 
+	/// The backend used by the node.
+	type Backend: crate::backend::Backend<B>;
+
 	/// Execute a call to a contract on top of state in a block of given hash.
 	///
 	/// No changes are made.
@@ -76,6 +71,9 @@ where
 		method: &str,
 		call_data: &[u8],
 		changes: &RefCell<OverlayedChanges>,
+		storage_transaction_cache: Option<&RefCell<
+			StorageTransactionCache<B, <Self::Backend as crate::backend::Backend<B>>::State>,
+		>>,
 		initialize_block: InitializeBlock<'a, B>,
 		execution_manager: ExecutionManager<EM>,
 		native_call: Option<NC>,
@@ -88,38 +86,10 @@ where
 	/// No changes are made.
 	fn runtime_version(&self, id: &BlockId<B>) -> Result<RuntimeVersion, sp_blockchain::Error>;
 
-	/// Execute a call to a contract on top of given state.
-	///
-	/// No changes are made.
-	fn call_at_state<
-		S: sp_state_machine::Backend<H>,
-		F: FnOnce(
-			Result<NativeOrEncoded<R>, Self::Error>,
-			Result<NativeOrEncoded<R>, Self::Error>,
-		) -> Result<NativeOrEncoded<R>, Self::Error>,
-		R: Encode + Decode + PartialEq,
-		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
-	>(&self,
-		state: &S,
-		overlay: &mut OverlayedChanges,
-		method: &str,
-		call_data: &[u8],
-		manager: ExecutionManager<F>,
-		native_call: Option<NC>,
-		extensions: Option<Extensions>,
-	) -> Result<
-		(
-			NativeOrEncoded<R>,
-			(S::Transaction, H::Out),
-			Option<ChangesTrieTransaction<Blake2Hasher, NumberFor<B>>>
-		),
-		sp_blockchain::Error,
-	>;
-
 	/// Execute a call to a contract on top of given state, gathering execution proof.
 	///
 	/// No changes are made.
-	fn prove_at_state<S: sp_state_machine::Backend<H>>(
+	fn prove_at_state<S: sp_state_machine::Backend<HasherFor<B>>>(
 		&self,
 		mut state: S,
 		overlay: &mut OverlayedChanges,
@@ -137,9 +107,9 @@ where
 	/// Execute a call to a contract on top of given trie state, gathering execution proof.
 	///
 	/// No changes are made.
-	fn prove_at_trie_state<S: sp_state_machine::TrieBackendStorage<H>>(
+	fn prove_at_trie_state<S: sp_state_machine::TrieBackendStorage<HasherFor<B>>>(
 		&self,
-		trie_state: &sp_state_machine::TrieBackend<S, H>,
+		trie_state: &sp_state_machine::TrieBackend<S, HasherFor<B>>,
 		overlay: &mut OverlayedChanges,
 		method: &str,
 		call_data: &[u8]

substrate/client/api/src/lib.rs

@@ -34,12 +34,10 @@ pub use notifications::*;
 
 pub use sp_state_machine::{StorageProof, ExecutionStrategy};
 
-
 /// Utility methods for the client.
 pub mod utils {
-    use sp_blockchain::{HeaderBackend, HeaderMetadata, Error};
-    use sp_core::H256;
-	use sp_runtime::traits::{Block as BlockT};
+	use sp_blockchain::{HeaderBackend, HeaderMetadata, Error};
+	use sp_runtime::traits::Block as BlockT;
 	use std::borrow::Borrow;
 
 	/// Returns a function for checking block ancestry, the returned function will
@@ -48,11 +46,11 @@ pub mod utils {
 	/// represent the current block `hash` and its `parent hash`, if given the
 	/// function that's returned will assume that `hash` isn't part of the local DB
 	/// yet, and all searches in the DB will instead reference the parent.
-	pub fn is_descendent_of<'a, Block: BlockT<Hash=H256>, T, H: Borrow<H256> + 'a>(
+	pub fn is_descendent_of<'a, Block: BlockT, T>(
 		client: &'a T,
-		current: Option<(H, H)>,
-	) -> impl Fn(&H256, &H256) -> Result<bool, Error> + 'a
-		where T: HeaderBackend<Block> + HeaderMetadata<Block, Error=Error>,
+		current: Option<(Block::Hash, Block::Hash)>,
+	) -> impl Fn(&Block::Hash, &Block::Hash) -> Result<bool, Error> + 'a
+		where T: HeaderBackend<Block> + HeaderMetadata<Block, Error = Error>,
 	{
 		move |base, hash| {
 			if base == hash { return Ok(false); }