Remove requirement on Hash = H256, make Proposer return StorageChanges and Proof (#3860)

* Extend `Proposer` to optionally generate a proof of the proposal

* Something

* Refactor sr-api to not depend on client anymore

* Fix benches

* Apply suggestions from code review

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Apply suggestions from code review

* Introduce new `into_storage_changes` function

* Switch to runtime api for `execute_block` and don't require `H256`
anywhere in the code

* Put the `StorageChanges` into the `Proposal`

* Move the runtime api error to its own trait

* Adds `StorageTransactionCache` to the runtime api

This requires that we add `type NodeBlock = ` to the
`impl_runtime_apis!` macro to work around some bugs in rustc :(

* Remove `type NodeBlock` and switch to a "better" hack

* Start using the transaction cache from the runtime api

* Make it compile

* Move `InMemory` to its own file

* Make all tests work again

* Return block, storage_changes and proof from Blockbuilder::bake()

* Make sure that we use/set `storage_changes` when possible

* Add test

* Fix deadlock

* Remove accidentally added folders

* Introduce `RecordProof` as argument type to be more explicit

* Update client/src/client.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update primitives/state-machine/src/ext.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Integrates review feedback

* Remove `unsafe` usage

* Update client/block-builder/src/lib.rs

Co-Authored-By: Benjamin Kampmann <ben@gnunicorn.org>

* Update client/src/call_executor.rs

* Bump versions

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>
Co-authored-by: Benjamin Kampmann <ben.kampmann@googlemail.com>

substrate/test-utils/runtime/client/src/trait_tests.rs

@@ -21,19 +21,22 @@
 
 use std::sync::Arc;
 
-use sc_client_api::backend::LocalBackend;
-use crate::block_builder_ext::BlockBuilderExt;
+use crate::{
+	AccountKeyring, ClientBlockImportExt, BlockBuilderExt, TestClientBuilder, TestClientBuilderExt,
+};
+use sc_client_api::backend;
 use sc_client_api::blockchain::{Backend as BlockChainBackendT, HeaderBackend};
-use crate::{AccountKeyring, ClientExt, TestClientBuilder, TestClientBuilderExt};
 use substrate_test_client::sp_consensus::BlockOrigin;
-use sp_core::Blake2Hasher;
 use substrate_test_runtime::{self, Transfer};
 use sp_runtime::generic::BlockId;
-use sp_runtime::traits::Block as BlockT;
+use sp_runtime::traits::{Block as BlockT, HasherFor};
 
 /// helper to test the `leaves` implementation for various backends
 pub fn test_leaves_for_backend<B: 'static>(backend: Arc<B>) where
-	B: LocalBackend<substrate_test_runtime::Block, Blake2Hasher>,
+	B: backend::Backend<substrate_test_runtime::Block>,
+	// Rust bug: https://github.com/rust-lang/rust/issues/24159
+	backend::StateBackendFor<B, substrate_test_runtime::Block>:
+		sp_api::StateBackend<HasherFor<substrate_test_runtime::Block>>,
 {
 	// block tree:
 	// G -> A1 -> A2 -> A3 -> A4 -> A5
@@ -41,7 +44,7 @@ pub fn test_leaves_for_backend<B: 'static>(backend: Arc<B>) where
 	//			  B2 -> C3
 	//		A1 -> D2
 
-	let client = TestClientBuilder::with_backend(backend.clone()).build();
+	let mut client = TestClientBuilder::with_backend(backend.clone()).build();
 	let blockchain = backend.blockchain();
 
 	let genesis_hash = client.chain_info().genesis_hash;
@@ -51,44 +54,72 @@ pub fn test_leaves_for_backend<B: 'static>(backend: Arc<B>) where
 		vec![genesis_hash]);
 
 	// G -> A1
-	let a1 = client.new_block(Default::default()).unwrap().bake().unwrap();
+	let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, a1.clone()).unwrap();
 	assert_eq!(
 		blockchain.leaves().unwrap(),
-		vec![a1.hash()]);
+		vec![a1.hash()],
+	);
 
 	// A1 -> A2
-	let a2 = client.new_block_at(&BlockId::Hash(a1.hash()), Default::default()).unwrap().bake().unwrap();
+	let a2 = client.new_block_at(
+		&BlockId::Hash(a1.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, a2.clone()).unwrap();
 
 	#[allow(deprecated)]
 	assert_eq!(
 		blockchain.leaves().unwrap(),
-		vec![a2.hash()]);
+		vec![a2.hash()],
+	);
 
 	// A2 -> A3
-	let a3 = client.new_block_at(&BlockId::Hash(a2.hash()), Default::default()).unwrap().bake().unwrap();
+	let a3 = client.new_block_at(
+		&BlockId::Hash(a2.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, a3.clone()).unwrap();
+
 	assert_eq!(
 		blockchain.leaves().unwrap(),
-		vec![a3.hash()]);
+		vec![a3.hash()],
+	);
 
 	// A3 -> A4
-	let a4 = client.new_block_at(&BlockId::Hash(a3.hash()), Default::default()).unwrap().bake().unwrap();
+	let a4 = client.new_block_at(
+		&BlockId::Hash(a3.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, a4.clone()).unwrap();
 	assert_eq!(
 		blockchain.leaves().unwrap(),
-		vec![a4.hash()]);
+		vec![a4.hash()],
+	);
 
 	// A4 -> A5
-	let a5 = client.new_block_at(&BlockId::Hash(a4.hash()), Default::default()).unwrap().bake().unwrap();
+	let a5 = client.new_block_at(
+		&BlockId::Hash(a4.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
+
 	client.import(BlockOrigin::Own, a5.clone()).unwrap();
 	assert_eq!(
 		blockchain.leaves().unwrap(),
-		vec![a5.hash()]);
+		vec![a5.hash()],
+	);
 
 	// A1 -> B2
-	let mut builder = client.new_block_at(&BlockId::Hash(a1.hash()), Default::default()).unwrap();
+	let mut builder = client.new_block_at(
+		&BlockId::Hash(a1.hash()),
+		Default::default(),
+		false,
+	).unwrap();
+
 	// this push is required as otherwise B2 has the same hash as A2 and won't get imported
 	builder.push_transfer(Transfer {
 		from: AccountKeyring::Alice.into(),
@@ -96,28 +127,44 @@ pub fn test_leaves_for_backend<B: 'static>(backend: Arc<B>) where
 		amount: 41,
 		nonce: 0,
 	}).unwrap();
-	let b2 = builder.bake().unwrap();
+	let b2 = builder.build().unwrap().block;
 	client.import(BlockOrigin::Own, b2.clone()).unwrap();
 	assert_eq!(
 		blockchain.leaves().unwrap(),
-		vec![a5.hash(), b2.hash()]);
+		vec![a5.hash(), b2.hash()],
+	);
 
 	// B2 -> B3
-	let b3 = client.new_block_at(&BlockId::Hash(b2.hash()), Default::default()).unwrap().bake().unwrap();
+	let b3 = client.new_block_at(
+		&BlockId::Hash(b2.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
+
 	client.import(BlockOrigin::Own, b3.clone()).unwrap();
 	assert_eq!(
 		blockchain.leaves().unwrap(),
-		vec![a5.hash(), b3.hash()]);
+		vec![a5.hash(), b3.hash()],
+	);
 
 	// B3 -> B4
-	let b4 = client.new_block_at(&BlockId::Hash(b3.hash()), Default::default()).unwrap().bake().unwrap();
+	let b4 = client.new_block_at(
+		&BlockId::Hash(b3.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, b4.clone()).unwrap();
 	assert_eq!(
 		blockchain.leaves().unwrap(),
-		vec![a5.hash(), b4.hash()]);
+		vec![a5.hash(), b4.hash()],
+	);
 
 	// // B2 -> C3
-	let mut builder = client.new_block_at(&BlockId::Hash(b2.hash()), Default::default()).unwrap();
+	let mut builder = client.new_block_at(
+		&BlockId::Hash(b2.hash()),
+		Default::default(),
+		false,
+	).unwrap();
 	// this push is required as otherwise C3 has the same hash as B3 and won't get imported
 	builder.push_transfer(Transfer {
 		from: AccountKeyring::Alice.into(),
@@ -125,14 +172,19 @@ pub fn test_leaves_for_backend<B: 'static>(backend: Arc<B>) where
 		amount: 1,
 		nonce: 1,
 	}).unwrap();
-	let c3 = builder.bake().unwrap();
+	let c3 = builder.build().unwrap().block;
 	client.import(BlockOrigin::Own, c3.clone()).unwrap();
 	assert_eq!(
 		blockchain.leaves().unwrap(),
-		vec![a5.hash(), b4.hash(), c3.hash()]);
+		vec![a5.hash(), b4.hash(), c3.hash()],
+	);
 
 	// A1 -> D2
-	let mut builder = client.new_block_at(&BlockId::Hash(a1.hash()), Default::default()).unwrap();
+	let mut builder = client.new_block_at(
+		&BlockId::Hash(a1.hash()),
+		Default::default(),
+		false,
+	).unwrap();
 	// this push is required as otherwise D2 has the same hash as B2 and won't get imported
 	builder.push_transfer(Transfer {
 		from: AccountKeyring::Alice.into(),
@@ -140,16 +192,20 @@ pub fn test_leaves_for_backend<B: 'static>(backend: Arc<B>) where
 		amount: 1,
 		nonce: 0,
 	}).unwrap();
-	let d2 = builder.bake().unwrap();
+	let d2 = builder.build().unwrap().block;
 	client.import(BlockOrigin::Own, d2.clone()).unwrap();
 	assert_eq!(
 		blockchain.leaves().unwrap(),
-		vec![a5.hash(), b4.hash(), c3.hash(), d2.hash()]);
+		vec![a5.hash(), b4.hash(), c3.hash(), d2.hash()],
+	);
 }
 
 /// helper to test the `children` implementation for various backends
 pub fn test_children_for_backend<B: 'static>(backend: Arc<B>) where
-	B: LocalBackend<substrate_test_runtime::Block, Blake2Hasher>,
+	B: backend::LocalBackend<substrate_test_runtime::Block>,
+	// Rust bug: https://github.com/rust-lang/rust/issues/24159
+	<B as backend::Backend<substrate_test_runtime::Block>>::State:
+		sp_api::StateBackend<HasherFor<substrate_test_runtime::Block>>,
 {
 	// block tree:
 	// G -> A1 -> A2 -> A3 -> A4 -> A5
@@ -157,31 +213,51 @@ pub fn test_children_for_backend<B: 'static>(backend: Arc<B>) where
 	//			  B2 -> C3
 	//		A1 -> D2
 
-	let client = TestClientBuilder::with_backend(backend.clone()).build();
+	let mut client = TestClientBuilder::with_backend(backend.clone()).build();
 	let blockchain = backend.blockchain();
 
 	// G -> A1
-	let a1 = client.new_block(Default::default()).unwrap().bake().unwrap();
+	let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, a1.clone()).unwrap();
 
 	// A1 -> A2
-	let a2 = client.new_block_at(&BlockId::Hash(a1.hash()), Default::default()).unwrap().bake().unwrap();
+	let a2 = client.new_block_at(
+		&BlockId::Hash(a1.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, a2.clone()).unwrap();
 
 	// A2 -> A3
-	let a3 = client.new_block_at(&BlockId::Hash(a2.hash()), Default::default()).unwrap().bake().unwrap();
+	let a3 = client.new_block_at(
+		&BlockId::Hash(a2.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, a3.clone()).unwrap();
 
 	// A3 -> A4
-	let a4 = client.new_block_at(&BlockId::Hash(a3.hash()), Default::default()).unwrap().bake().unwrap();
+	let a4 = client.new_block_at(
+		&BlockId::Hash(a3.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, a4.clone()).unwrap();
 
 	// A4 -> A5
-	let a5 = client.new_block_at(&BlockId::Hash(a4.hash()), Default::default()).unwrap().bake().unwrap();
+	let a5 = client.new_block_at(
+		&BlockId::Hash(a4.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, a5.clone()).unwrap();
 
 	// A1 -> B2
-	let mut builder = client.new_block_at(&BlockId::Hash(a1.hash()), Default::default()).unwrap();
+	let mut builder = client.new_block_at(
+		&BlockId::Hash(a1.hash()),
+		Default::default(),
+		false,
+	).unwrap();
 	// this push is required as otherwise B2 has the same hash as A2 and won't get imported
 	builder.push_transfer(Transfer {
 		from: AccountKeyring::Alice.into(),
@@ -189,19 +265,31 @@ pub fn test_children_for_backend<B: 'static>(backend: Arc<B>) where
 		amount: 41,
 		nonce: 0,
 	}).unwrap();
-	let b2 = builder.bake().unwrap();
+	let b2 = builder.build().unwrap().block;
 	client.import(BlockOrigin::Own, b2.clone()).unwrap();
 
 	// B2 -> B3
-	let b3 = client.new_block_at(&BlockId::Hash(b2.hash()), Default::default()).unwrap().bake().unwrap();
+	let b3 = client.new_block_at(
+		&BlockId::Hash(b2.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, b3.clone()).unwrap();
 
 	// B3 -> B4
-	let b4 = client.new_block_at(&BlockId::Hash(b3.hash()), Default::default()).unwrap().bake().unwrap();
+	let b4 = client.new_block_at(
+		&BlockId::Hash(b3.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, b4.clone()).unwrap();
 
 	// // B2 -> C3
-	let mut builder = client.new_block_at(&BlockId::Hash(b2.hash()), Default::default()).unwrap();
+	let mut builder = client.new_block_at(
+		&BlockId::Hash(b2.hash()),
+		Default::default(),
+		false,
+	).unwrap();
 	// this push is required as otherwise C3 has the same hash as B3 and won't get imported
 	builder.push_transfer(Transfer {
 		from: AccountKeyring::Alice.into(),
@@ -209,11 +297,15 @@ pub fn test_children_for_backend<B: 'static>(backend: Arc<B>) where
 		amount: 1,
 		nonce: 1,
 	}).unwrap();
-	let c3 = builder.bake().unwrap();
+	let c3 = builder.build().unwrap().block;
 	client.import(BlockOrigin::Own, c3.clone()).unwrap();
 
 	// A1 -> D2
-	let mut builder = client.new_block_at(&BlockId::Hash(a1.hash()), Default::default()).unwrap();
+	let mut builder = client.new_block_at(
+		&BlockId::Hash(a1.hash()),
+		Default::default(),
+		false,
+	).unwrap();
 	// this push is required as otherwise D2 has the same hash as B2 and won't get imported
 	builder.push_transfer(Transfer {
 		from: AccountKeyring::Alice.into(),
@@ -221,7 +313,7 @@ pub fn test_children_for_backend<B: 'static>(backend: Arc<B>) where
 		amount: 1,
 		nonce: 0,
 	}).unwrap();
-	let d2 = builder.bake().unwrap();
+	let d2 = builder.build().unwrap().block;
 	client.import(BlockOrigin::Own, d2.clone()).unwrap();
 
 	let genesis_hash = client.chain_info().genesis_hash;
@@ -240,38 +332,61 @@ pub fn test_children_for_backend<B: 'static>(backend: Arc<B>) where
 }
 
 pub fn test_blockchain_query_by_number_gets_canonical<B: 'static>(backend: Arc<B>) where
-	B: LocalBackend<substrate_test_runtime::Block, Blake2Hasher>,
+	B: backend::LocalBackend<substrate_test_runtime::Block>,
+	// Rust bug: https://github.com/rust-lang/rust/issues/24159
+	<B as backend::Backend<substrate_test_runtime::Block>>::State:
+		sp_api::StateBackend<HasherFor<substrate_test_runtime::Block>>,
 {
 	// block tree:
 	// G -> A1 -> A2 -> A3 -> A4 -> A5
 	//		A1 -> B2 -> B3 -> B4
 	//			  B2 -> C3
 	//		A1 -> D2
-	let client = TestClientBuilder::with_backend(backend.clone()).build();
+	let mut client = TestClientBuilder::with_backend(backend.clone()).build();
 	let blockchain = backend.blockchain();
 
 	// G -> A1
-	let a1 = client.new_block(Default::default()).unwrap().bake().unwrap();
+	let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, a1.clone()).unwrap();
 
 	// A1 -> A2
-	let a2 = client.new_block_at(&BlockId::Hash(a1.hash()), Default::default()).unwrap().bake().unwrap();
+	let a2 = client.new_block_at(
+		&BlockId::Hash(a1.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, a2.clone()).unwrap();
 
 	// A2 -> A3
-	let a3 = client.new_block_at(&BlockId::Hash(a2.hash()), Default::default()).unwrap().bake().unwrap();
+	let a3 = client.new_block_at(
+		&BlockId::Hash(a2.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, a3.clone()).unwrap();
 
 	// A3 -> A4
-	let a4 = client.new_block_at(&BlockId::Hash(a3.hash()), Default::default()).unwrap().bake().unwrap();
+	let a4 = client.new_block_at(
+		&BlockId::Hash(a3.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, a4.clone()).unwrap();
 
 	// A4 -> A5
-	let a5 = client.new_block_at(&BlockId::Hash(a4.hash()), Default::default()).unwrap().bake().unwrap();
+	let a5 = client.new_block_at(
+		&BlockId::Hash(a4.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, a5.clone()).unwrap();
 
 	// A1 -> B2
-	let mut builder = client.new_block_at(&BlockId::Hash(a1.hash()), Default::default()).unwrap();
+	let mut builder = client.new_block_at(
+		&BlockId::Hash(a1.hash()),
+		Default::default(),
+		false,
+	).unwrap();
 	// this push is required as otherwise B2 has the same hash as A2 and won't get imported
 	builder.push_transfer(Transfer {
 		from: AccountKeyring::Alice.into(),
@@ -279,19 +394,31 @@ pub fn test_blockchain_query_by_number_gets_canonical<B: 'static>(backend: Arc<B
 		amount: 41,
 		nonce: 0,
 	}).unwrap();
-	let b2 = builder.bake().unwrap();
+	let b2 = builder.build().unwrap().block;
 	client.import(BlockOrigin::Own, b2.clone()).unwrap();
 
 	// B2 -> B3
-	let b3 = client.new_block_at(&BlockId::Hash(b2.hash()), Default::default()).unwrap().bake().unwrap();
+	let b3 = client.new_block_at(
+		&BlockId::Hash(b2.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, b3.clone()).unwrap();
 
 	// B3 -> B4
-	let b4 = client.new_block_at(&BlockId::Hash(b3.hash()), Default::default()).unwrap().bake().unwrap();
+	let b4 = client.new_block_at(
+		&BlockId::Hash(b3.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
 	client.import(BlockOrigin::Own, b4.clone()).unwrap();
 
 	// // B2 -> C3
-	let mut builder = client.new_block_at(&BlockId::Hash(b2.hash()), Default::default()).unwrap();
+	let mut builder = client.new_block_at(
+		&BlockId::Hash(b2.hash()),
+		Default::default(),
+		false,
+	).unwrap();
 	// this push is required as otherwise C3 has the same hash as B3 and won't get imported
 	builder.push_transfer(Transfer {
 		from: AccountKeyring::Alice.into(),
@@ -299,11 +426,15 @@ pub fn test_blockchain_query_by_number_gets_canonical<B: 'static>(backend: Arc<B
 		amount: 1,
 		nonce: 1,
 	}).unwrap();
-	let c3 = builder.bake().unwrap();
+	let c3 = builder.build().unwrap().block;
 	client.import(BlockOrigin::Own, c3.clone()).unwrap();
 
 	// A1 -> D2
-	let mut builder = client.new_block_at(&BlockId::Hash(a1.hash()), Default::default()).unwrap();
+	let mut builder = client.new_block_at(
+		&BlockId::Hash(a1.hash()),
+		Default::default(),
+		false,
+	).unwrap();
 	// this push is required as otherwise D2 has the same hash as B2 and won't get imported
 	builder.push_transfer(Transfer {
 		from: AccountKeyring::Alice.into(),
@@ -311,7 +442,7 @@ pub fn test_blockchain_query_by_number_gets_canonical<B: 'static>(backend: Arc<B
 		amount: 1,
 		nonce: 0,
 	}).unwrap();
-	let d2 = builder.bake().unwrap();
+	let d2 = builder.build().unwrap().block;
 	client.import(BlockOrigin::Own, d2.clone()).unwrap();
 
 	let genesis_hash = client.chain_info().genesis_hash;