Fix import queue thread pool shutdown (#4929)

* Fix import queue thread pool shutdown

* Make sure runtime is disposed before telemetry

* Close channel istead of sending a message

* Fixed test

substrate/client/cli/src/runtime.rs

@@ -128,12 +128,14 @@ where
 
 	// we eagerly drop the service so that the internal exit future is fired,
 	// but we need to keep holding a reference to the global telemetry guard
+	// and drop the runtime first.
 	let _telemetry = service.telemetry();
 
 	let f = service.fuse();
 	pin_mut!(f);
 
 	runtime.block_on(main(f)).map_err(|e| e.to_string())?;
+	drop(runtime);
 
 	Ok(())
 }

substrate/client/consensus/babe/src/tests.rs

@@ -229,7 +229,7 @@ impl Verifier<TestBlock> for TestVerifier {
 	) -> Result<(BlockImportParams<TestBlock, ()>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
 		// apply post-sealing mutations (i.e. stripping seal, if desired).
 		(self.mutator)(&mut header, Stage::PostSeal);
-		Ok(self.inner.verify(origin, header, justification, body).expect("verification failed!"))
+		self.inner.verify(origin, header, justification, body)
 	}
 }
 
@@ -423,7 +423,14 @@ fn run_one_test(
 	}
 
 	runtime.spawn(futures01::future::poll_fn(move || {
-		net.lock().poll();
+		let mut net = net.lock();
+		net.poll();
+		for p in net.peers() {
+			for (h, e) in p.failed_verifications() {
+				panic!("Verification failed for {:?}: {}", h, e);
+			}
+		}
+
 		Ok::<_, ()>(futures01::Async::NotReady::<()>)
 	}));
 

substrate/client/network/test/src/lib.rs

@@ -183,7 +183,7 @@ pub struct Peer<D, S: NetworkSpecialization<Block>> {
 	client: PeersClient,
 	/// We keep a copy of the verifier so that we can invoke it for locally-generated blocks,
 	/// instead of going through the import queue.
-	verifier: VerifierAdapter<dyn Verifier<Block>>,
+	verifier: VerifierAdapter<Block>,
 	/// We keep a copy of the block_import so that we can invoke it for locally-generated blocks,
 	/// instead of going through the import queue.
 	block_import: BlockImportAdapter<()>,
@@ -360,6 +360,11 @@ impl<D, S: NetworkSpecialization<Block>> Peer<D, S> {
 			|backend| backend.blocks_count()
 		).unwrap_or(0)
 	}
+
+	/// Return a collection of block hashes that failed verification
+	pub fn failed_verifications(&self) -> HashMap<<Block as BlockT>::Hash, String> {
+		self.verifier.failed_verifications.lock().clone()
+	}
 }
 
 pub struct EmptyTransactionPool;
@@ -485,15 +490,13 @@ impl<Transaction> BlockImport<Block> for BlockImportAdapter<Transaction> {
 }
 
 /// Implements `Verifier` on an `Arc<Mutex<impl Verifier>>`. Used internally.
-struct VerifierAdapter<T: ?Sized>(Arc<Mutex<Box<T>>>);
-
-impl<T: ?Sized> Clone for VerifierAdapter<T> {
-	fn clone(&self) -> Self {
-		VerifierAdapter(self.0.clone())
-	}
+#[derive(Clone)]
+struct VerifierAdapter<B: BlockT> {
+	verifier: Arc<Mutex<Box<dyn Verifier<B>>>>,
+	failed_verifications: Arc<Mutex<HashMap<B::Hash, String>>>,
 }
 
-impl<B: BlockT, T: ?Sized + Verifier<B>> Verifier<B> for VerifierAdapter<T> {
+impl<B: BlockT> Verifier<B> for VerifierAdapter<B> {
 	fn verify(
 		&mut self,
 		origin: BlockOrigin,
@@ -501,7 +504,20 @@ impl<B: BlockT, T: ?Sized + Verifier<B>> Verifier<B> for VerifierAdapter<T> {
 		justification: Option<Justification>,
 		body: Option<Vec<B::Extrinsic>>
 	) -> Result<(BlockImportParams<B, ()>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
-		self.0.lock().verify(origin, header, justification, body)
+		let hash = header.hash();
+		self.verifier.lock().verify(origin, header, justification, body).map_err(|e| {
+			self.failed_verifications.lock().insert(hash, e.clone());
+			e
+		})
+	}
+}
+
+impl<B: BlockT> VerifierAdapter<B> {
+	fn new(verifier: Arc<Mutex<Box<dyn Verifier<B>>>>) -> VerifierAdapter<B> {
+		VerifierAdapter {
+			verifier,
+			failed_verifications: Default::default(),
+		}
 	}
 }
 
@@ -592,7 +608,7 @@ pub trait TestNetFactory: Sized {
 			config,
 			&data,
 		);
-		let verifier = VerifierAdapter(Arc::new(Mutex::new(Box::new(verifier) as Box<_>)));
+		let verifier = VerifierAdapter::new(Arc::new(Mutex::new(Box::new(verifier) as Box<_>)));
 
 		let import_queue = Box::new(BasicQueue::new(
 			verifier.clone(),
@@ -668,7 +684,7 @@ pub trait TestNetFactory: Sized {
 			&config,
 			&data,
 		);
-		let verifier = VerifierAdapter(Arc::new(Mutex::new(Box::new(verifier) as Box<_>)));
+		let verifier = VerifierAdapter::new(Arc::new(Mutex::new(Box::new(verifier) as Box<_>)));
 
 		let import_queue = Box::new(BasicQueue::new(
 			verifier.clone(),