Remove requirement on Hash = H256, make Proposer return StorageChanges and Proof (#3860)

* Extend `Proposer` to optionally generate a proof of the proposal

* Something

* Refactor sr-api to not depend on client anymore

* Fix benches

* Apply suggestions from code review

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Apply suggestions from code review

* Introduce new `into_storage_changes` function

* Switch to runtime api for `execute_block` and don't require `H256`
anywhere in the code

* Put the `StorageChanges` into the `Proposal`

* Move the runtime api error to its own trait

* Adds `StorageTransactionCache` to the runtime api

This requires that we add `type NodeBlock = ` to the
`impl_runtime_apis!` macro to work around some bugs in rustc :(

* Remove `type NodeBlock` and switch to a "better" hack

* Start using the transaction cache from the runtime api

* Make it compile

* Move `InMemory` to its own file

* Make all tests work again

* Return block, storage_changes and proof from Blockbuilder::bake()

* Make sure that we use/set `storage_changes` when possible

* Add test

* Fix deadlock

* Remove accidentally added folders

* Introduce `RecordProof` as argument type to be more explicit

* Update client/src/client.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update primitives/state-machine/src/ext.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Integrates review feedback

* Remove `unsafe` usage

* Update client/block-builder/src/lib.rs

Co-Authored-By: Benjamin Kampmann <ben@gnunicorn.org>

* Update client/src/call_executor.rs

* Bump versions

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>
Co-authored-by: Benjamin Kampmann <ben.kampmann@googlemail.com>

substrate/bin/node/cli/src/service.rs

@@ -39,7 +39,6 @@ use sp_runtime::traits::Block as BlockT;
 use node_executor::NativeExecutor;
 use sc_network::NetworkService;
 use sc_offchain::OffchainWorkers;
-use sp_core::Blake2Hasher;
 
 construct_simple_protocol! {
 	/// Demo protocol attachment for substrate.
@@ -113,9 +112,7 @@ macro_rules! new_full_start {
 /// concrete types instead.
 macro_rules! new_full {
 	($config:expr, $with_startup_data: expr) => {{
-		use futures01::Stream;
 		use futures::{
-			compat::Stream01CompatExt,
 			stream::StreamExt,
 			future::{FutureExt, TryFutureExt},
 		};
@@ -300,7 +297,7 @@ pub fn new_full<C: Send + Default + 'static>(config: NodeConfiguration<C>)
 		ConcreteTransactionPool,
 		OffchainWorkers<
 			ConcreteClient,
-			<ConcreteBackend as sc_client_api::backend::Backend<Block, Blake2Hasher>>::OffchainStorage,
+			<ConcreteBackend as sc_client_api::backend::Backend<Block>>::OffchainStorage,
 			ConcreteBlock,
 		>
 	>,
@@ -387,6 +384,7 @@ mod tests {
 	use sc_consensus_babe::CompatibleDigestItem;
 	use sp_consensus::{
 		Environment, Proposer, BlockImportParams, BlockOrigin, ForkChoiceStrategy, BlockImport,
+		RecordProof,
 	};
 	use node_primitives::{Block, DigestItem, Signature};
 	use node_runtime::{BalancesCall, Call, UncheckedExtrinsic, Address};
@@ -439,6 +437,7 @@ mod tests {
 				internal_justification: Vec::new(),
 				finalized: false,
 				body: Some(block.extrinsics),
+				storage_changes: None,
 				header: block.header,
 				auxiliary: Vec::new(),
 			}
@@ -540,7 +539,8 @@ mod tests {
 					inherent_data,
 					digest,
 					std::time::Duration::from_secs(1),
-				)).expect("Error making test block");
+					RecordProof::Yes,
+				)).expect("Error making test block").block;
 
 				let (new_header, new_body) = new_block.deconstruct();
 				let pre_hash = new_header.hash();
@@ -559,6 +559,7 @@ mod tests {
 					justification: None,
 					post_digests: vec![item],
 					body: Some(new_body),
+					storage_changes: None,
 					finalized: false,
 					auxiliary: Vec::new(),
 					fork_choice: ForkChoiceStrategy::LongestChain,

substrate/bin/node/executor/Cargo.toml

@@ -37,7 +37,7 @@ wabt = "0.9.2"
 
 [features]
 wasmtime = [
-    "sc-executor/wasmtime",
+	"sc-executor/wasmtime",
 ]
 wasmi-errno = [
 	"sc-executor/wasmi-errno",

substrate/bin/node/executor/src/lib.rs

@@ -27,4 +27,3 @@ native_executor_instance!(
 	node_runtime::api::dispatch,
 	node_runtime::native_version
 );
-

substrate/bin/node/executor/tests/basic.rs

@@ -43,7 +43,7 @@ use node_primitives::{Balance, Hash};
 use wabt;
 use node_testing::keyring::*;
 
-mod common;
+pub mod common;
 use self::common::{*, sign};
 
 /// The wasm runtime binary which hasn't undergone the compacting process.
@@ -821,11 +821,11 @@ fn full_wasm_block_import_works_with_changes_trie() {
 #[test]
 fn should_import_block_with_test_client() {
 	use node_testing::client::{
-		ClientExt, TestClientBuilderExt, TestClientBuilder,
+		ClientBlockImportExt, TestClientBuilderExt, TestClientBuilder,
 		sp_consensus::BlockOrigin,
 	};
 
-	let client = TestClientBuilder::new().build();
+	let mut client = TestClientBuilder::new().build();
 	let block1 = changes_trie_block();
 	let block_data = block1.0;
 	let block = node_primitives::Block::decode(&mut &block_data[..]).unwrap();

substrate/bin/node/executor/tests/common.rs

@@ -15,15 +15,13 @@
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
 use codec::{Encode, Decode};
-use frame_support::{
-	Hashable,
-};
+use frame_support::Hashable;
 use sp_state_machine::TestExternalities as CoreTestExternalities;
 use sp_core::{
 	Blake2Hasher, NeverNativeValue, NativeOrEncoded,
 	traits::CodeExecutor,
 };
-use sp_runtime::traits::{Header as HeaderT};
+use sp_runtime::traits::Header as HeaderT;
 use sc_executor::{NativeExecutor, WasmExecutionMethod};
 use sc_executor::error::Result;
 

substrate/bin/node/executor/tests/fees.rs

@@ -37,7 +37,7 @@ use node_runtime::impls::LinearWeightToFee;
 use node_primitives::Balance;
 use node_testing::keyring::*;
 
-mod common;
+pub mod common;
 use self::common::{*, sign};
 
 #[test]

substrate/bin/node/executor/tests/submit_transaction.rs

@@ -28,7 +28,7 @@ use frame_system::offchain::{SubmitSignedTransaction, SubmitUnsignedTransaction}
 use pallet_im_online::sr25519::AuthorityPair as Key;
 use codec::Decode;
 
-mod common;
+pub mod common;
 use self::common::*;
 
 #[test]

substrate/bin/node/rpc/Cargo.toml

@@ -10,6 +10,7 @@ jsonrpc-core = "14.0.3"
 node-primitives = { version = "2.0.0", path = "../primitives" }
 node-runtime = { version = "2.0.0", path = "../runtime" }
 sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" }
+sp-api = { version = "2.0.0", path = "../../../primitives/api" }
 pallet-contracts-rpc = { version = "2.0.0", path = "../../../frame/contracts/rpc/" }
 pallet-transaction-payment-rpc = { version = "2.0.0", path = "../../../frame/transaction-payment/rpc/" }
 substrate-frame-rpc-system = { version = "2.0.0", path = "../../../utils/frame/rpc/system" }

substrate/bin/node/rpc/src/lib.rs

@@ -33,7 +33,7 @@ use std::sync::Arc;
 
 use node_primitives::{Block, AccountId, Index, Balance};
 use node_runtime::UncheckedExtrinsic;
-use sp_runtime::traits::ProvideRuntimeApi;
+use sp_api::ProvideRuntimeApi;
 use sp_transaction_pool::TransactionPool;
 
 /// Light client extra dependencies.
@@ -62,7 +62,7 @@ pub fn create<C, P, M, F>(
 	pool: Arc<P>,
 	light_deps: Option<LightDeps<F>>,
 ) -> jsonrpc_core::IoHandler<M> where
-	C: ProvideRuntimeApi,
+	C: ProvideRuntimeApi<Block>,
 	C: sc_client::blockchain::HeaderBackend<Block>,
 	C: Send + Sync + 'static,
 	C::Api: substrate_frame_rpc_system::AccountNonceApi<Block, AccountId, Index>,

substrate/bin/node/runtime/src/lib.rs

@@ -80,8 +80,8 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
 	// and set impl_version to equal spec_version. If only runtime
 	// implementation changes and behavior does not, then leave spec_version as
 	// is and increment impl_version.
-	spec_version: 200,
-	impl_version: 200,
+	spec_version: 201,
+	impl_version: 201,
 	apis: RUNTIME_API_VERSIONS,
 };
 

substrate/bin/node/transaction-factory/src/complex_mode.rs

@@ -43,10 +43,9 @@ use std::sync::Arc;
 use log::info;
 use sc_client::Client;
 use sp_block_builder::BlockBuilder;
-use sp_api::ConstructRuntimeApi;
-use sp_core::{Blake2Hasher, Hasher};
+use sp_api::{ConstructRuntimeApi, ProvideRuntimeApi};
 use sp_runtime::generic::BlockId;
-use sp_runtime::traits::{Block as BlockT, ProvideRuntimeApi, One, Zero};
+use sp_runtime::traits::{Block as BlockT, One, Zero};
 
 use crate::{RuntimeAdapter, create_block};
 
@@ -59,12 +58,13 @@ pub fn next<RA, Backend, Exec, Block, RtApi>(
 	prior_block_id: BlockId<Block>,
 ) -> Option<Block>
 where
-	Block: BlockT<Hash = <Blake2Hasher as Hasher>::Out>,
-	Exec: sc_client::CallExecutor<Block, Blake2Hasher> + Send + Sync + Clone,
-	Backend: sc_client_api::backend::Backend<Block, Blake2Hasher> + Send,
-	Client<Backend, Exec, Block, RtApi>: ProvideRuntimeApi,
-	<Client<Backend, Exec, Block, RtApi> as ProvideRuntimeApi>::Api:
-		BlockBuilder<Block, Error = sp_blockchain::Error>,
+	Block: BlockT,
+	Exec: sc_client::CallExecutor<Block, Backend = Backend> + Send + Sync + Clone,
+	Backend: sc_client_api::backend::Backend<Block> + Send,
+	Client<Backend, Exec, Block, RtApi>: ProvideRuntimeApi<Block>,
+	<Client<Backend, Exec, Block, RtApi> as ProvideRuntimeApi<Block>>::Api:
+		BlockBuilder<Block, Error = sp_blockchain::Error> +
+		sp_api::ApiExt<Block, StateBackend = Backend::State>,
 	RtApi: ConstructRuntimeApi<Block, Client<Backend, Exec, Block, RtApi>> + Send + Sync,
 	RA: RuntimeAdapter,
 {

substrate/bin/node/transaction-factory/src/lib.rs

@@ -28,18 +28,16 @@ use log::info;
 
 use sc_client::Client;
 use sp_block_builder::BlockBuilder;
-use sp_api::ConstructRuntimeApi;
+use sp_api::{ConstructRuntimeApi, ProvideRuntimeApi, ApiExt};
 use sp_consensus::{
 	BlockOrigin, BlockImportParams, InherentData, ForkChoiceStrategy,
 	SelectChain
 };
 use sp_consensus::block_import::BlockImport;
 use codec::{Decode, Encode};
-use sp_core::{Blake2Hasher, Hasher};
 use sp_runtime::generic::BlockId;
 use sp_runtime::traits::{
-	Block as BlockT, Header as HeaderT, ProvideRuntimeApi, SimpleArithmetic,
-	One, Zero,
+	Block as BlockT, Header as HeaderT, SimpleArithmetic, One, Zero,
 };
 pub use crate::modes::Mode;
 
@@ -100,16 +98,17 @@ pub fn factory<RA, Backend, Exec, Block, RtApi, Sc>(
 	select_chain: &Sc,
 ) -> sc_cli::error::Result<()>
 where
-	Block: BlockT<Hash = <Blake2Hasher as Hasher>::Out>,
-	Exec: sc_client::CallExecutor<Block, Blake2Hasher> + Send + Sync + Clone,
-	Backend: sc_client_api::backend::Backend<Block, Blake2Hasher> + Send,
-	Client<Backend, Exec, Block, RtApi>: ProvideRuntimeApi,
-	<Client<Backend, Exec, Block, RtApi> as ProvideRuntimeApi>::Api:
-		BlockBuilder<Block, Error = sp_blockchain::Error>,
+	Block: BlockT,
+	Exec: sc_client::CallExecutor<Block, Backend = Backend> + Send + Sync + Clone,
+	Backend: sc_client_api::backend::Backend<Block> + Send,
+	Client<Backend, Exec, Block, RtApi>: ProvideRuntimeApi<Block>,
+	<Client<Backend, Exec, Block, RtApi> as ProvideRuntimeApi<Block>>::Api:
+		BlockBuilder<Block, Error = sp_blockchain::Error> +
+		ApiExt<Block, StateBackend = Backend::State>,
 	RtApi: ConstructRuntimeApi<Block, Client<Backend, Exec, Block, RtApi>> + Send + Sync,
 	Sc: SelectChain<Block>,
-	RA: RuntimeAdapter,
-	<<RA as RuntimeAdapter>::Block as BlockT>::Hash: From<sp_core::H256>,
+	RA: RuntimeAdapter<Block = Block>,
+	Block::Hash: From<sp_core::H256>,
 {
 	if *factory_state.mode() != Mode::MasterToNToM && factory_state.rounds() > RA::Number::one() {
 		let msg = "The factory can only be used with rounds set to 1 in this mode.".into();
@@ -144,7 +143,7 @@ where
 	} {
 		best_hash = block.header().hash();
 		best_block_id = BlockId::<Block>::hash(best_hash);
-		import_block(&client, block);
+		import_block(client.clone(), block);
 
 		info!("Imported block at {}", factory_state.block_no());
 	}
@@ -159,13 +158,14 @@ pub fn create_block<RA, Backend, Exec, Block, RtApi>(
 	inherent_extrinsics: Vec<<Block as BlockT>::Extrinsic>,
 ) -> Block
 where
-	Block: BlockT<Hash = <Blake2Hasher as Hasher>::Out>,
-	Exec: sc_client::CallExecutor<Block, Blake2Hasher> + Send + Sync + Clone,
-	Backend: sc_client_api::backend::Backend<Block, Blake2Hasher> + Send,
-	Client<Backend, Exec, Block, RtApi>: ProvideRuntimeApi,
+	Block: BlockT,
+	Exec: sc_client::CallExecutor<Block, Backend = Backend> + Send + Sync + Clone,
+	Backend: sc_client_api::backend::Backend<Block> + Send,
+	Client<Backend, Exec, Block, RtApi>: ProvideRuntimeApi<Block>,
 	RtApi: ConstructRuntimeApi<Block, Client<Backend, Exec, Block, RtApi>> + Send + Sync,
-	<Client<Backend, Exec, Block, RtApi> as ProvideRuntimeApi>::Api:
-		BlockBuilder<Block, Error = sp_blockchain::Error>,
+	<Client<Backend, Exec, Block, RtApi> as ProvideRuntimeApi<Block>>::Api:
+		BlockBuilder<Block, Error = sp_blockchain::Error> +
+		ApiExt<Block, StateBackend = Backend::State>,
 	RA: RuntimeAdapter,
 {
 	let mut block = client.new_block(Default::default()).expect("Failed to create new block");
@@ -178,22 +178,27 @@ where
 		block.push(inherent).expect("Failed ...");
 	}
 
-	block.bake().expect("Failed to bake block")
+	block.build().expect("Failed to bake block").block
 }
 
 fn import_block<Backend, Exec, Block, RtApi>(
-	client: &Arc<Client<Backend, Exec, Block, RtApi>>,
+	mut client: Arc<Client<Backend, Exec, Block, RtApi>>,
 	block: Block
 ) -> () where
-	Block: BlockT<Hash = <Blake2Hasher as Hasher>::Out>,
-	Exec: sc_client::CallExecutor<Block, Blake2Hasher> + Send + Sync + Clone,
-	Backend: sc_client_api::backend::Backend<Block, Blake2Hasher> + Send,
+	Block: BlockT,
+	Exec: sc_client::CallExecutor<Block> + Send + Sync + Clone,
+	Backend: sc_client_api::backend::Backend<Block> + Send,
+	Client<Backend, Exec, Block, RtApi>: ProvideRuntimeApi<Block>,
+	<Client<Backend, Exec, Block, RtApi> as ProvideRuntimeApi<Block>>::Api:
+		sp_api::Core<Block, Error = sp_blockchain::Error> +
+		ApiExt<Block, StateBackend = Backend::State>,
 {
 	let import = BlockImportParams {
 		origin: BlockOrigin::File,
 		header: block.header().clone(),
 		post_digests: Vec::new(),
 		body: Some(block.extrinsics().to_vec()),
+		storage_changes: None,
 		finalized: false,
 		justification: None,
 		auxiliary: Vec::new(),
@@ -201,5 +206,5 @@ fn import_block<Backend, Exec, Block, RtApi>(
 		allow_missing_state: false,
 		import_existing: false,
 	};
-	(&**client).import_block(import, HashMap::new()).expect("Failed to import block");
+	client.import_block(import, HashMap::new()).expect("Failed to import block");
 }

substrate/bin/node/transaction-factory/src/simple_modes.rs

@@ -38,9 +38,8 @@ use std::sync::Arc;
 use log::info;
 use sc_client::Client;
 use sp_block_builder::BlockBuilder;
-use sp_api::ConstructRuntimeApi;
-use sp_core::{Blake2Hasher, Hasher};
-use sp_runtime::traits::{Block as BlockT, ProvideRuntimeApi, One};
+use sp_api::{ConstructRuntimeApi, ProvideRuntimeApi};
+use sp_runtime::traits::{Block as BlockT, One};
 use sp_runtime::generic::BlockId;
 
 use crate::{Mode, RuntimeAdapter, create_block};
@@ -54,12 +53,13 @@ pub fn next<RA, Backend, Exec, Block, RtApi>(
 	prior_block_id: BlockId<Block>,
 ) -> Option<Block>
 where
-	Block: BlockT<Hash = <Blake2Hasher as Hasher>::Out>,
-	Exec: sc_client::CallExecutor<Block, Blake2Hasher> + Send + Sync + Clone,
-	Backend: sc_client_api::backend::Backend<Block, Blake2Hasher> + Send,
-	Client<Backend, Exec, Block, RtApi>: ProvideRuntimeApi,
-	<Client<Backend, Exec, Block, RtApi> as ProvideRuntimeApi>::Api:
-		BlockBuilder<Block, Error = sp_blockchain::Error>,
+	Block: BlockT,
+	Exec: sc_client::CallExecutor<Block, Backend = Backend> + Send + Sync + Clone,
+	Backend: sc_client_api::backend::Backend<Block> + Send,
+	Client<Backend, Exec, Block, RtApi>: ProvideRuntimeApi<Block>,
+	<Client<Backend, Exec, Block, RtApi> as ProvideRuntimeApi<Block>>::Api:
+		BlockBuilder<Block, Error = sp_blockchain::Error> +
+		sp_api::ApiExt<Block, StateBackend = Backend::State>,
 	RtApi: ConstructRuntimeApi<Block, Client<Backend, Exec, Block, RtApi>> + Send + Sync,
 	RA: RuntimeAdapter,
 {