Remove requirement on Hash = H256, make Proposer return StorageChanges and Proof (#3860)

* Extend `Proposer` to optionally generate a proof of the proposal

* Something

* Refactor sr-api to not depend on client anymore

* Fix benches

* Apply suggestions from code review

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Apply suggestions from code review

* Introduce new `into_storage_changes` function

* Switch to runtime api for `execute_block` and don't require `H256`
anywhere in the code

* Put the `StorageChanges` into the `Proposal`

* Move the runtime api error to its own trait

* Adds `StorageTransactionCache` to the runtime api

This requires that we add `type NodeBlock = ` to the
`impl_runtime_apis!` macro to work around some bugs in rustc :(

* Remove `type NodeBlock` and switch to a "better" hack

* Start using the transaction cache from the runtime api

* Make it compile

* Move `InMemory` to its own file

* Make all tests work again

* Return block, storage_changes and proof from Blockbuilder::bake()

* Make sure that we use/set `storage_changes` when possible

* Add test

* Fix deadlock

* Remove accidentally added folders

* Introduce `RecordProof` as argument type to be more explicit

* Update client/src/client.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update primitives/state-machine/src/ext.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Integrates review feedback

* Remove `unsafe` usage

* Update client/block-builder/src/lib.rs

Co-Authored-By: Benjamin Kampmann <ben@gnunicorn.org>

* Update client/src/call_executor.rs

* Bump versions

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>
Co-authored-by: Benjamin Kampmann <ben.kampmann@googlemail.com>

substrate/primitives/consensus/common/src/block_import.rs

@@ -16,8 +16,10 @@
 
 //! Block import helpers.
 
-use sp_runtime::traits::{Block as BlockT, DigestItemFor, Header as HeaderT, NumberFor};
-use sp_runtime::Justification;
+use sp_runtime::{
+	Justification,
+	traits::{Block as BlockT, DigestItemFor, Header as HeaderT, NumberFor, HasherFor},
+};
 use std::borrow::Cow;
 use std::collections::HashMap;
 use std::sync::Arc;
@@ -110,7 +112,7 @@ pub struct BlockCheckParams<Block: BlockT> {
 }
 
 /// Data required to import a Block.
-pub struct BlockImportParams<Block: BlockT> {
+pub struct BlockImportParams<Block: BlockT, Transaction> {
 	/// Origin of the Block
 	pub origin: BlockOrigin,
 	/// The header, without consensus post-digests applied. This should be in the same
@@ -130,8 +132,13 @@ pub struct BlockImportParams<Block: BlockT> {
 	/// Digest items that have been added after the runtime for external
 	/// work, like a consensus signature.
 	pub post_digests: Vec<DigestItemFor<Block>>,
-	/// Block's body
+	/// The body of the block.
 	pub body: Option<Vec<Block::Extrinsic>>,
+	/// The changes to the storage to create the state for the block. If this is `Some(_)`,
+	/// the block import will not need to re-execute the block for importing it.
+	pub storage_changes: Option<
+		sp_state_machine::StorageChanges<Transaction, HasherFor<Block>, NumberFor<Block>>
+	>,
 	/// Is this block finalized already?
 	/// `true` implies instant finality.
 	pub finalized: bool,
@@ -148,7 +155,7 @@ pub struct BlockImportParams<Block: BlockT> {
 	pub import_existing: bool,
 }
 
-impl<Block: BlockT> BlockImportParams<Block> {
+impl<Block: BlockT, Transaction> BlockImportParams<Block, Transaction> {
 	/// Deconstruct the justified header into parts.
 	pub fn into_inner(self)
 		-> (
@@ -156,7 +163,8 @@ impl<Block: BlockT> BlockImportParams<Block> {
 			<Block as BlockT>::Header,
 			Option<Justification>,
 			Vec<DigestItemFor<Block>>,
-			Option<Vec<<Block as BlockT>::Extrinsic>>,
+			Option<Vec<Block::Extrinsic>>,
+			Option<sp_state_machine::StorageChanges<Transaction, HasherFor<Block>, NumberFor<Block>>>,
 			bool,
 			Vec<(Vec<u8>, Option<Vec<u8>>)>,
 		) {
@@ -166,6 +174,7 @@ impl<Block: BlockT> BlockImportParams<Block> {
 			self.justification,
 			self.post_digests,
 			self.body,
+			self.storage_changes,
 			self.finalized,
 			self.auxiliary,
 		)
@@ -186,11 +195,34 @@ impl<Block: BlockT> BlockImportParams<Block> {
 			})
 		}
 	}
+
+	/// Auxiliary function for "converting" the transaction type.
+	///
+	/// Actually this just sets `storage_changes` to `None` and makes rustc think that `Self` now
+	/// uses a different transaction type.
+	pub fn convert_transaction<Transaction2>(self) -> BlockImportParams<Block, Transaction2> {
+		BlockImportParams {
+			origin: self.origin,
+			header: self.header,
+			justification: self.justification,
+			post_digests: self.post_digests,
+			body: self.body,
+			storage_changes: None,
+			finalized: self.finalized,
+			auxiliary: self.auxiliary,
+			allow_missing_state: self.allow_missing_state,
+			fork_choice: self.fork_choice,
+			import_existing: self.import_existing,
+		}
+	}
 }
 
 /// Block import trait.
 pub trait BlockImport<B: BlockT> {
-	type Error: ::std::error::Error + Send + 'static;
+	/// The error type.
+	type Error: std::error::Error + Send + 'static;
+	/// The transaction type used by the backend.
+	type Transaction;
 
 	/// Check block preconditions.
 	fn check_block(
@@ -203,13 +235,14 @@ pub trait BlockImport<B: BlockT> {
 	/// Cached data can be accessed through the blockchain cache.
 	fn import_block(
 		&mut self,
-		block: BlockImportParams<B>,
+		block: BlockImportParams<B, Self::Transaction>,
 		cache: HashMap<CacheKeyId, Vec<u8>>,
 	) -> Result<ImportResult, Self::Error>;
 }
 
-impl<B: BlockT> BlockImport<B> for crate::import_queue::BoxBlockImport<B> {
+impl<B: BlockT, Transaction> BlockImport<B> for crate::import_queue::BoxBlockImport<B, Transaction> {
 	type Error = crate::error::Error;
+	type Transaction = Transaction;
 
 	/// Check block preconditions.
 	fn check_block(
@@ -224,17 +257,18 @@ impl<B: BlockT> BlockImport<B> for crate::import_queue::BoxBlockImport<B> {
 	/// Cached data can be accessed through the blockchain cache.
 	fn import_block(
 		&mut self,
-		block: BlockImportParams<B>,
+		block: BlockImportParams<B, Transaction>,
 		cache: HashMap<CacheKeyId, Vec<u8>>,
 	) -> Result<ImportResult, Self::Error> {
 		(**self).import_block(block, cache)
 	}
 }
 
-impl<B: BlockT, T, E: std::error::Error + Send + 'static> BlockImport<B> for Arc<T>
-where for<'r> &'r T: BlockImport<B, Error = E>
+impl<B: BlockT, T, E: std::error::Error + Send + 'static, Transaction> BlockImport<B> for Arc<T>
+	where for<'r> &'r T: BlockImport<B, Error = E, Transaction = Transaction>
 {
 	type Error = E;
+	type Transaction = Transaction;
 
 	fn check_block(
 		&mut self,
@@ -245,7 +279,7 @@ where for<'r> &'r T: BlockImport<B, Error = E>
 
 	fn import_block(
 		&mut self,
-		block: BlockImportParams<B>,
+		block: BlockImportParams<B, Transaction>,
 		cache: HashMap<CacheKeyId, Vec<u8>>,
 	) -> Result<ImportResult, Self::Error> {
 		(&**self).import_block(block, cache)
@@ -254,7 +288,7 @@ where for<'r> &'r T: BlockImport<B, Error = E>
 
 /// Justification import trait
 pub trait JustificationImport<B: BlockT> {
-	type Error: ::std::error::Error + Send + 'static;
+	type Error: std::error::Error + Send + 'static;
 
 	/// Called by the import queue when it is started. Returns a list of justifications to request
 	/// from the network.

substrate/primitives/consensus/common/src/import_queue.rs

@@ -39,13 +39,17 @@ mod basic_queue;
 pub mod buffered_link;
 
 /// Shared block import struct used by the queue.
-pub type BoxBlockImport<B> = Box<dyn BlockImport<B, Error = ConsensusError> + Send + Sync>;
+pub type BoxBlockImport<B, Transaction> = Box<
+	dyn BlockImport<B, Error = ConsensusError, Transaction = Transaction> + Send + Sync
+>;
 
 /// Shared justification import struct used by the queue.
 pub type BoxJustificationImport<B> = Box<dyn JustificationImport<B, Error=ConsensusError> + Send + Sync>;
 
 /// Shared finality proof import struct used by the queue.
-pub type BoxFinalityProofImport<B> = Box<dyn FinalityProofImport<B, Error=ConsensusError> + Send + Sync>;
+pub type BoxFinalityProofImport<B> = Box<
+	dyn FinalityProofImport<B, Error = ConsensusError> + Send + Sync
+>;
 
 /// Maps to the Origin used by the network.
 pub type Origin = libp2p::PeerId;
@@ -83,7 +87,7 @@ pub trait Verifier<B: BlockT>: Send + Sync {
 		header: B::Header,
 		justification: Option<Justification>,
 		body: Option<Vec<B::Extrinsic>>,
-	) -> Result<(BlockImportParams<B>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String>;
+	) -> Result<(BlockImportParams<B, ()>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String>;
 }
 
 /// Blocks import queue API.
@@ -176,8 +180,8 @@ pub enum BlockImportError {
 }
 
 /// Single block import function.
-pub fn import_single_block<B: BlockT, V: Verifier<B>>(
-	import_handle: &mut dyn BlockImport<B, Error = ConsensusError>,
+pub fn import_single_block<B: BlockT, V: Verifier<B>, Transaction>(
+	import_handle: &mut dyn BlockImport<B, Transaction = Transaction, Error = ConsensusError>,
 	block_origin: BlockOrigin,
 	block: IncomingBlock<B>,
 	verifier: &mut V,
@@ -254,5 +258,5 @@ pub fn import_single_block<B: BlockT, V: Verifier<B>>(
 	}
 	import_block.allow_missing_state = block.allow_missing_state;
 
-	import_error(import_handle.import_block(import_block, cache))
+	import_error(import_handle.import_block(import_block.convert_transaction(), cache))
 }

substrate/primitives/consensus/common/src/import_queue/basic_queue.rs

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::{mem, pin::Pin, time::Duration};
+use std::{mem, pin::Pin, time::Duration, marker::PhantomData};
 use futures::{prelude::*, channel::mpsc, task::Context, task::Poll};
 use futures_timer::Delay;
 use sp_runtime::{Justification, traits::{Block as BlockT, Header as HeaderT, NumberFor}};
@@ -29,7 +29,7 @@ use crate::import_queue::{
 
 /// Interface to a basic block import queue that is importing blocks sequentially in a separate
 /// task, with pluggable verification.
-pub struct BasicQueue<B: BlockT> {
+pub struct BasicQueue<B: BlockT, Transaction> {
 	/// Channel to send messages to the background task.
 	sender: mpsc::UnboundedSender<ToWorkerMsg<B>>,
 	/// Results coming from the worker task.
@@ -40,16 +40,17 @@ pub struct BasicQueue<B: BlockT> {
 	manual_poll: Option<Pin<Box<dyn Future<Output = ()> + Send>>>,
 	/// A thread pool where the background worker is being run.
 	pool: Option<futures::executor::ThreadPool>,
+	_phantom: PhantomData<Transaction>,
 }
 
-impl<B: BlockT> BasicQueue<B> {
+impl<B: BlockT, Transaction: Send + 'static> BasicQueue<B, Transaction> {
 	/// Instantiate a new basic queue, with given verifier.
 	///
 	/// This creates a background task, and calls `on_start` on the justification importer and
 	/// finality proof importer.
 	pub fn new<V: 'static + Verifier<B>>(
 		verifier: V,
-		block_import: BoxBlockImport<B>,
+		block_import: BoxBlockImport<B, Transaction>,
 		justification_import: Option<BoxJustificationImport<B>>,
 		finality_proof_import: Option<BoxFinalityProofImport<B>>,
 	) -> Self {
@@ -81,11 +82,12 @@ impl<B: BlockT> BasicQueue<B> {
 			result_port,
 			manual_poll,
 			pool,
+			_phantom: PhantomData,
 		}
 	}
 }
 
-impl<B: BlockT> ImportQueue<B> for BasicQueue<B> {
+impl<B: BlockT, Transaction: Send> ImportQueue<B> for BasicQueue<B, Transaction> {
 	fn import_blocks(&mut self, origin: BlockOrigin, blocks: Vec<IncomingBlock<B>>) {
 		if blocks.is_empty() {
 			return;
@@ -102,12 +104,24 @@ impl<B: BlockT> ImportQueue<B> for BasicQueue<B> {
 		number: NumberFor<B>,
 		justification: Justification
 	) {
-		let _ = self.sender.unbounded_send(ToWorkerMsg::ImportJustification(who.clone(), hash, number, justification));
+		let _ = self.sender
+			.unbounded_send(
+				ToWorkerMsg::ImportJustification(who.clone(), hash, number, justification)
+			);
 	}
 
-	fn import_finality_proof(&mut self, who: Origin, hash: B::Hash, number: NumberFor<B>, finality_proof: Vec<u8>) {
+	fn import_finality_proof(
+		&mut self,
+		who: Origin,
+		hash: B::Hash,
+		number: NumberFor<B>,
+		finality_proof: Vec<u8>,
+	) {
 		trace!(target: "sync", "Scheduling finality proof of {}/{} for import", number, hash);
-		let _ = self.sender.unbounded_send(ToWorkerMsg::ImportFinalityProof(who, hash, number, finality_proof));
+		let _ = self.sender
+			.unbounded_send(
+				ToWorkerMsg::ImportFinalityProof(who, hash, number, finality_proof)
+			);
 	}
 
 	fn poll_actions(&mut self, cx: &mut Context, link: &mut dyn Link<B>) {
@@ -132,18 +146,19 @@ enum ToWorkerMsg<B: BlockT> {
 	ImportFinalityProof(Origin, B::Hash, NumberFor<B>, Vec<u8>),
 }
 
-struct BlockImportWorker<B: BlockT> {
+struct BlockImportWorker<B: BlockT, Transaction> {
 	result_sender: BufferedLinkSender<B>,
 	justification_import: Option<BoxJustificationImport<B>>,
 	finality_proof_import: Option<BoxFinalityProofImport<B>>,
 	delay_between_blocks: Duration,
+	_phantom: PhantomData<Transaction>,
 }
 
-impl<B: BlockT> BlockImportWorker<B> {
+impl<B: BlockT, Transaction: Send> BlockImportWorker<B, Transaction> {
 	fn new<V: 'static + Verifier<B>>(
 		result_sender: BufferedLinkSender<B>,
 		verifier: V,
-		block_import: BoxBlockImport<B>,
+		block_import: BoxBlockImport<B, Transaction>,
 		justification_import: Option<BoxJustificationImport<B>>,
 		finality_proof_import: Option<BoxFinalityProofImport<B>>,
 	) -> (impl Future<Output = ()> + Send, mpsc::UnboundedSender<ToWorkerMsg<B>>) {
@@ -154,6 +169,7 @@ impl<B: BlockT> BlockImportWorker<B> {
 			justification_import,
 			finality_proof_import,
 			delay_between_blocks: Duration::new(0, 0),
+			_phantom: PhantomData,
 		};
 
 		// Let's initialize `justification_import` and `finality_proof_import`.
@@ -237,11 +253,11 @@ impl<B: BlockT> BlockImportWorker<B> {
 	/// yielded back in the output once the import is finished.
 	fn import_a_batch_of_blocks<V: 'static + Verifier<B>>(
 		&mut self,
-		block_import: BoxBlockImport<B>,
+		block_import: BoxBlockImport<B, Transaction>,
 		verifier: V,
 		origin: BlockOrigin,
 		blocks: Vec<IncomingBlock<B>>
-	) -> impl Future<Output = (BoxBlockImport<B>, V)> {
+	) -> impl Future<Output = (BoxBlockImport<B, Transaction>, V)> {
 		let mut result_sender = self.result_sender.clone();
 
 		import_many_blocks(block_import, origin, blocks, verifier, self.delay_between_blocks)
@@ -309,16 +325,22 @@ impl<B: BlockT> BlockImportWorker<B> {
 ///
 /// The returned `Future` yields at every imported block, which makes the execution more
 /// fine-grained and making it possible to interrupt the process.
-fn import_many_blocks<B: BlockT, V: Verifier<B>>(
-	import_handle: BoxBlockImport<B>,
+fn import_many_blocks<B: BlockT, V: Verifier<B>, Transaction>(
+	import_handle: BoxBlockImport<B, Transaction>,
 	blocks_origin: BlockOrigin,
 	blocks: Vec<IncomingBlock<B>>,
 	verifier: V,
 	delay_between_blocks: Duration,
-) -> impl Future<Output = (usize, usize, Vec<(
-	Result<BlockImportResult<NumberFor<B>>, BlockImportError>,
-	B::Hash,
-)>, BoxBlockImport<B>, V)> {
+) -> impl Future<
+	Output = (
+		usize,
+		usize,
+		Vec<(Result<BlockImportResult<NumberFor<B>>, BlockImportError>, B::Hash,)>,
+		BoxBlockImport<B, Transaction>,
+		V
+	)
+>
+{
 	let count = blocks.len();
 
 	let blocks_range = match (

substrate/primitives/consensus/common/src/lib.rs

@@ -31,7 +31,9 @@
 use std::sync::Arc;
 use std::time::Duration;
 
-use sp_runtime::{traits::{Block as BlockT, DigestFor}, generic::BlockId};
+use sp_runtime::{
+	generic::BlockId, traits::{Block as BlockT, DigestFor, NumberFor, HasherFor},
+};
 use futures::prelude::*;
 pub use sp_inherents::InherentData;
 
@@ -48,10 +50,11 @@ const MAX_BLOCK_SIZE: usize = 4 * 1024 * 1024 + 512;
 
 pub use self::error::Error;
 pub use block_import::{
-	BlockImport, BlockOrigin, ForkChoiceStrategy, ImportedAux, BlockImportParams, BlockCheckParams, ImportResult,
-	JustificationImport, FinalityProofImport,
+	BlockImport, BlockOrigin, ForkChoiceStrategy, ImportedAux, BlockImportParams, BlockCheckParams,
+	ImportResult, JustificationImport, FinalityProofImport,
 };
 pub use select_chain::SelectChain;
+pub use sp_state_machine::Backend as StateBackend;
 
 /// Block status.
 #[derive(Debug, PartialEq, Eq)]
@@ -71,14 +74,56 @@ pub enum BlockStatus {
 /// Environment producer for a Consensus instance. Creates proposer instance and communication streams.
 pub trait Environment<B: BlockT> {
 	/// The proposer type this creates.
-	type Proposer: Proposer<B>;
+	type Proposer: Proposer<B> + 'static;
 	/// Error which can occur upon creation.
-	type Error: From<Error>;
+	type Error: From<Error> + std::fmt::Debug + 'static;
 
 	/// Initialize the proposal logic on top of a specific header. Provide
 	/// the authorities at that header.
-	fn init(&mut self, parent_header: &B::Header)
-		-> Result<Self::Proposer, Self::Error>;
+	fn init(&mut self, parent_header: &B::Header) -> Result<Self::Proposer, Self::Error>;
+}
+
+/// A proposal that is created by a [`Proposer`].
+pub struct Proposal<Block: BlockT, Transaction> {
+	/// The block that was build.
+	pub block: Block,
+	/// Optional proof that was recorded while building the block.
+	pub proof: Option<sp_state_machine::StorageProof>,
+	/// The storage changes while building this block.
+	pub storage_changes: sp_state_machine::StorageChanges<Transaction, HasherFor<Block>, NumberFor<Block>>,
+}
+
+/// Used as parameter to [`Proposer`] to tell the requirement on recording a proof.
+///
+/// When `RecordProof::Yes` is given, all accessed trie nodes should be saved. These recorded
+/// trie nodes can be used by a third party to proof this proposal without having access to the
+/// full storage.
+#[derive(Copy, Clone, PartialEq)]
+pub enum RecordProof {
+	/// `Yes`, record a proof.
+	Yes,
+	/// `No`, don't record any proof.
+	No,
+}
+
+impl RecordProof {
+	/// Returns if `Self` == `Yes`.
+	pub fn yes(&self) -> bool {
+		match self {
+			Self::Yes => true,
+			Self::No => false,
+		}
+	}
+}
+
+impl From<bool> for RecordProof {
+	fn from(val: bool) -> Self {
+		if val {
+			Self::Yes
+		} else {
+			Self::No
+		}
+	}
 }
 
 /// Logic for a proposer.
@@ -89,16 +134,29 @@ pub trait Environment<B: BlockT> {
 /// Proposers are generic over bits of "consensus data" which are engine-specific.
 pub trait Proposer<B: BlockT> {
 	/// Error type which can occur when proposing or evaluating.
-	type Error: From<Error> + ::std::fmt::Debug + 'static;
-	/// Future that resolves to a committed proposal.
-	type Create: Future<Output = Result<B, Self::Error>>;
+	type Error: From<Error> + std::fmt::Debug + 'static;
+	/// The transaction type used by the backend.
+	type Transaction: Default + Send + 'static;
+	/// Future that resolves to a committed proposal with an optional proof.
+	type Proposal: Future<Output = Result<Proposal<B, Self::Transaction>, Self::Error>> +
+		Send + Unpin + 'static;
+
 	/// Create a proposal.
+	///
+	/// Gets the `inherent_data` and `inherent_digests` as input for the proposal. Additionally
+	/// a maximum duration for building this proposal is given. If building the proposal takes
+	/// longer than this maximum, the proposal will be very likely discarded.
+	///
+	/// # Return
+	///
+	/// Returns a future that resolves to a [`Proposal`] or to [`Self::Error`].
 	fn propose(
 		&mut self,
 		inherent_data: InherentData,
 		inherent_digests: DigestFor<B>,
 		max_duration: Duration,
-	) -> Self::Create;
+		record_proof: RecordProof,
+	) -> Self::Proposal;
 }
 
 /// An oracle for when major synchronization work is being undertaken.