Make BlockImport and Verifier async (#8472)

* Make grandpa work

* Introduce `SharedData`

* Add test and fix bugs

* Switch to `SharedData`

* Make grandpa tests working

* More Babe work

* Make it async

* Fix fix

* Use `async_trait` in sc-consensus-slots

This makes the code a little bit easier to read and also expresses that
there can always only be one call at a time to `on_slot`.

* Make grandpa tests compile

* More Babe tests work

* Fix network test

* Start fixing service test

* Finish service-test

* Fix sc-consensus-aura

* Fix fix fix

* More fixes

* Make everything compile *yeah*

* Fix build when we have Rust 1.51

* Update client/consensus/common/src/shared_data.rs

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

* Update client/consensus/common/src/shared_data.rs

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

* Update client/consensus/common/src/shared_data.rs

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

* Update client/consensus/common/src/shared_data.rs

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

* Update client/consensus/common/src/shared_data.rs

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

* Update client/consensus/babe/src/tests.rs

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

* Update client/consensus/babe/src/tests.rs

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

* Fix warning

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

substrate/primitives/consensus/common/src/block_import.rs

@@ -146,7 +146,7 @@ pub struct BlockImportParams<Block: BlockT, Transaction> {
 	/// Intermediate values that are interpreted by block importers. Each block importer,
 	/// upon handling a value, removes it from the intermediate list. The final block importer
 	/// rejects block import if there are still intermediate values that remain unhandled.
-	pub intermediates: HashMap<Cow<'static, [u8]>, Box<dyn Any>>,
+	pub intermediates: HashMap<Cow<'static, [u8]>, Box<dyn Any + Send>>,
 	/// Auxiliary consensus data produced by the block.
 	/// Contains a list of key-value pairs. If values are `None`, the keys
 	/// will be deleted.
@@ -264,14 +264,15 @@ impl<Block: BlockT, Transaction> BlockImportParams<Block, Transaction> {
 }
 
 /// Block import trait.
+#[async_trait::async_trait]
 pub trait BlockImport<B: BlockT> {
 	/// The error type.
 	type Error: std::error::Error + Send + 'static;
 	/// The transaction type used by the backend.
-	type Transaction;
+	type Transaction: Send + 'static;
 
 	/// Check block preconditions.
-	fn check_block(
+	async fn check_block(
 		&mut self,
 		block: BlockCheckParams<B>,
 	) -> Result<ImportResult, Self::Error>;
@@ -279,56 +280,64 @@ pub trait BlockImport<B: BlockT> {
 	/// Import a block.
 	///
 	/// Cached data can be accessed through the blockchain cache.
-	fn import_block(
+	async fn import_block(
 		&mut self,
 		block: BlockImportParams<B, Self::Transaction>,
 		cache: HashMap<CacheKeyId, Vec<u8>>,
 	) -> Result<ImportResult, Self::Error>;
 }
 
-impl<B: BlockT, Transaction> BlockImport<B> for crate::import_queue::BoxBlockImport<B, Transaction> {
+#[async_trait::async_trait]
+impl<B: BlockT, Transaction> BlockImport<B> for crate::import_queue::BoxBlockImport<B, Transaction>
+	where
+		Transaction: Send + 'static,
+{
 	type Error = crate::error::Error;
 	type Transaction = Transaction;
 
 	/// Check block preconditions.
-	fn check_block(
+	async fn check_block(
 		&mut self,
 		block: BlockCheckParams<B>,
 	) -> Result<ImportResult, Self::Error> {
-		(**self).check_block(block)
+		(**self).check_block(block).await
 	}
 
 	/// Import a block.
 	///
 	/// Cached data can be accessed through the blockchain cache.
-	fn import_block(
+	async fn import_block(
 		&mut self,
 		block: BlockImportParams<B, Transaction>,
 		cache: HashMap<CacheKeyId, Vec<u8>>,
 	) -> Result<ImportResult, Self::Error> {
-		(**self).import_block(block, cache)
+		(**self).import_block(block, cache).await
 	}
 }
 
+#[async_trait::async_trait]
 impl<B: BlockT, T, E: std::error::Error + Send + 'static, Transaction> BlockImport<B> for Arc<T>
-	where for<'r> &'r T: BlockImport<B, Error = E, Transaction = Transaction>
+	where
+		for<'r> &'r T: BlockImport<B, Error = E, Transaction = Transaction>,
+		T: Send + Sync,
+		Transaction: Send + 'static,
 {
 	type Error = E;
 	type Transaction = Transaction;
 
-	fn check_block(
+	async fn check_block(
 		&mut self,
 		block: BlockCheckParams<B>,
 	) -> Result<ImportResult, Self::Error> {
-		(&**self).check_block(block)
+		(&**self).check_block(block).await
 	}
 
-	fn import_block(
+	async fn import_block(
 		&mut self,
 		block: BlockImportParams<B, Transaction>,
 		cache: HashMap<CacheKeyId, Vec<u8>>,
 	) -> Result<ImportResult, Self::Error> {
-		(&**self).import_block(block, cache)
+		(&**self).import_block(block, cache).await
 	}
 }
 

substrate/primitives/consensus/common/src/import_queue.rs

@@ -82,11 +82,12 @@ pub struct IncomingBlock<B: BlockT> {
 pub type CacheKeyId = [u8; 4];
 
 /// Verify a justification of a block
+#[async_trait::async_trait]
 pub trait Verifier<B: BlockT>: Send + Sync {
 	/// Verify the given data and return the BlockImportParams and an optional
 	/// new set of validators to import. If not, err with an Error-Message
 	/// presented to the User in the logs.
-	fn verify(
+	async fn verify(
 		&mut self,
 		origin: BlockOrigin,
 		header: B::Header,
@@ -163,18 +164,18 @@ pub enum BlockImportError {
 }
 
 /// Single block import function.
-pub fn import_single_block<B: BlockT, V: Verifier<B>, Transaction>(
-	import_handle: &mut dyn BlockImport<B, Transaction = Transaction, Error = ConsensusError>,
+pub async fn import_single_block<B: BlockT, V: Verifier<B>, Transaction: Send + 'static>(
+	import_handle: &mut impl BlockImport<B, Transaction = Transaction, Error = ConsensusError>,
 	block_origin: BlockOrigin,
 	block: IncomingBlock<B>,
 	verifier: &mut V,
 ) -> Result<BlockImportResult<NumberFor<B>>, BlockImportError> {
-	import_single_block_metered(import_handle, block_origin, block, verifier, None)
+	import_single_block_metered(import_handle, block_origin, block, verifier, None).await
 }
 
 /// Single block import function with metering.
-pub(crate) fn import_single_block_metered<B: BlockT, V: Verifier<B>, Transaction>(
-	import_handle: &mut dyn BlockImport<B, Transaction = Transaction, Error = ConsensusError>,
+pub(crate) async fn import_single_block_metered<B: BlockT, V: Verifier<B>, Transaction: Send + 'static>(
+	import_handle: &mut impl BlockImport<B, Transaction = Transaction, Error = ConsensusError>,
 	block_origin: BlockOrigin,
 	block: IncomingBlock<B>,
 	verifier: &mut V,
@@ -232,24 +233,28 @@ pub(crate) fn import_single_block_metered<B: BlockT, V: Verifier<B>, Transaction
 		parent_hash,
 		allow_missing_state: block.allow_missing_state,
 		import_existing: block.import_existing,
-	}))? {
+	}).await)? {
 		BlockImportResult::ImportedUnknown { .. } => (),
 		r => return Ok(r), // Any other successful result means that the block is already imported.
 	}
 
 	let started = wasm_timer::Instant::now();
-	let (mut import_block, maybe_keys) = verifier.verify(block_origin, header, justifications, block.body)
-		.map_err(|msg| {
-			if let Some(ref peer) = peer {
-				trace!(target: "sync", "Verifying {}({}) from {} failed: {}", number, hash, peer, msg);
-			} else {
-				trace!(target: "sync", "Verifying {}({}) failed: {}", number, hash, msg);
-			}
-			if let Some(metrics) = metrics.as_ref() {
-				metrics.report_verification(false, started.elapsed());
-			}
-			BlockImportError::VerificationFailed(peer.clone(), msg)
-		})?;
+	let (mut import_block, maybe_keys) = verifier.verify(
+		block_origin,
+		header,
+		justifications,
+		block.body
+	).await.map_err(|msg| {
+		if let Some(ref peer) = peer {
+			trace!(target: "sync", "Verifying {}({}) from {} failed: {}", number, hash, peer, msg);
+		} else {
+			trace!(target: "sync", "Verifying {}({}) failed: {}", number, hash, msg);
+		}
+		if let Some(metrics) = metrics.as_ref() {
+			metrics.report_verification(false, started.elapsed());
+		}
+		BlockImportError::VerificationFailed(peer.clone(), msg)
+	})?;
 
 	if let Some(metrics) = metrics.as_ref() {
 		metrics.report_verification(true, started.elapsed());
@@ -261,7 +266,7 @@ pub(crate) fn import_single_block_metered<B: BlockT, V: Verifier<B>, Transaction
 	}
 	import_block.allow_missing_state = block.allow_missing_state;
 
-	let imported = import_handle.import_block(import_block.convert_transaction(), cache);
+	let imported = import_handle.import_block(import_block.convert_transaction(), cache).await;
 	if let Some(metrics) = metrics.as_ref() {
 		metrics.report_verification_and_import(started.elapsed());
 	}

substrate/primitives/consensus/common/src/import_queue/basic_queue.rs

@@ -155,7 +155,7 @@ mod worker_messages {
 /// to be run.
 ///
 /// Returns when `block_import` ended.
-async fn block_import_process<B: BlockT, Transaction: Send>(
+async fn block_import_process<B: BlockT, Transaction: Send + 'static>(
 	mut block_import: BoxBlockImport<B, Transaction>,
 	mut verifier: impl Verifier<B>,
 	mut result_sender: BufferedLinkSender<B>,
@@ -195,7 +195,7 @@ struct BlockImportWorker<B: BlockT> {
 }
 
 impl<B: BlockT> BlockImportWorker<B> {
-	fn new<V: 'static + Verifier<B>, Transaction: Send>(
+	fn new<V: 'static + Verifier<B>, Transaction: Send + 'static>(
 		result_sender: BufferedLinkSender<B>,
 		verifier: V,
 		block_import: BoxBlockImport<B, Transaction>,
@@ -322,7 +322,7 @@ struct ImportManyBlocksResult<B: BlockT> {
 /// Import several blocks at once, returning import result for each block.
 ///
 /// This will yield after each imported block once, to ensure that other futures can be called as well.
-async fn import_many_blocks<B: BlockT, V: Verifier<B>, Transaction>(
+async fn import_many_blocks<B: BlockT, V: Verifier<B>, Transaction: Send + 'static>(
 	import_handle: &mut BoxBlockImport<B, Transaction>,
 	blocks_origin: BlockOrigin,
 	blocks: Vec<IncomingBlock<B>>,
@@ -371,7 +371,7 @@ async fn import_many_blocks<B: BlockT, V: Verifier<B>, Transaction>(
 				block,
 				verifier,
 				metrics.clone(),
-			)
+			).await
 		};
 
 		if let Some(metrics) = metrics.as_ref() {
@@ -439,8 +439,9 @@ mod tests {
 	use sp_test_primitives::{Block, BlockNumber, Extrinsic, Hash, Header};
 	use std::collections::HashMap;
 
+	#[async_trait::async_trait]
 	impl Verifier<Block> for () {
-		fn verify(
+		async fn verify(
 			&mut self,
 			origin: BlockOrigin,
 			header: Header,
@@ -451,18 +452,19 @@ mod tests {
 		}
 	}
 
+	#[async_trait::async_trait]
 	impl BlockImport<Block> for () {
 		type Error = crate::Error;
 		type Transaction = Extrinsic;
 
-		fn check_block(
+		async fn check_block(
 			&mut self,
 			_block: BlockCheckParams<Block>,
 		) -> Result<ImportResult, Self::Error> {
 			Ok(ImportResult::imported(false))
 		}
 
-		fn import_block(
+		async fn import_block(
 			&mut self,
 			_block: BlockImportParams<Block, Self::Transaction>,
 			_cache: HashMap<CacheKeyId, Vec<u8>>,