Split NetworkBridge and break cycles with Unbounded (#2736)

* overseer: pass messages directly between subsystems

* test that message is held on to

* Update node/overseer/src/lib.rs

Co-authored-by: Peter Goodspeed-Niklaus <coriolinus@users.noreply.github.com>

* give every subsystem an unbounded sender too

* remove metered_channel::name

1. we don't provide good names
2. these names are never used anywhere

* unused mut

* remove unnecessary &mut

* subsystem unbounded_send

* remove unused MaybeTimer

We have channel size metrics that serve the same purpose better now and the implementation of message timing was pretty ugly.

* remove comment

* split up senders and receivers

* update metrics

* fix tests

* fix test subsystem context

* use SubsystemSender in jobs system now

* refactor of awful jobs code

* expose public `run` on JobSubsystem

* update candidate backing to new jobs & use unbounded

* bitfield signing

* candidate-selection

* provisioner

* approval voting: send unbounded for assignment/approvals

* async not needed

* begin bridge split

* split up network tasks into background worker

* port over network bridge

* Update node/network/bridge/src/lib.rs

Co-authored-by: Andronik Ordian <write@reusable.software>

* rename ValidationWorkerNotifications

Co-authored-by: Peter Goodspeed-Niklaus <coriolinus@users.noreply.github.com>
Co-authored-by: Andronik Ordian <write@reusable.software>

polkadot/node/core/approval-voting/src/lib.rs

@@ -564,10 +564,10 @@ async fn handle_actions(
 				let block_hash = indirect_cert.block_hash;
 				let validator_index = indirect_cert.validator;
 
-				ctx.send_message(ApprovalDistributionMessage::DistributeAssignment(
+				ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeAssignment(
 					indirect_cert,
 					candidate_index,
-				).into()).await;
+				).into());
 
 				launch_approval(
 					ctx,
@@ -712,7 +712,7 @@ async fn handle_background_request(
 ) -> SubsystemResult<Vec<Action>> {
 	match request {
 		BackgroundRequest::ApprovalVote(vote_request) => {
-			issue_approval(ctx, state, metrics, vote_request).await
+			issue_approval(ctx, state, metrics, vote_request)
 		}
 		BackgroundRequest::CandidateValidation(
 			validation_data,
@@ -1724,7 +1724,7 @@ async fn launch_approval(
 
 // Issue and import a local approval vote. Should only be invoked after approval checks
 // have been done.
-async fn issue_approval(
+fn issue_approval(
 	ctx: &mut impl SubsystemContext,
 	state: &State<impl DBReader>,
 	metrics: &Metrics,
@@ -1830,12 +1830,14 @@ async fn issue_approval(
 	metrics.on_approval_produced();
 
 	// dispatch to approval distribution.
-	ctx.send_message(ApprovalDistributionMessage::DistributeApproval(IndirectSignedApprovalVote {
-		block_hash,
-		candidate_index: candidate_index as _,
-		validator: validator_index,
-		signature: sig,
-	}).into()).await;
+	ctx.send_unbounded_message(
+		ApprovalDistributionMessage::DistributeApproval(IndirectSignedApprovalVote {
+			block_hash,
+			candidate_index: candidate_index as _,
+			validator: validator_index,
+			signature: sig,
+		}
+	).into());
 
 	Ok(actions)
 }

polkadot/node/core/backing/src/lib.rs

@@ -35,7 +35,8 @@ use polkadot_node_primitives::{
 	Statement, SignedFullStatement, ValidationResult,
 };
 use polkadot_subsystem::{
-	PerLeafSpan, Stage, jaeger,
+	PerLeafSpan, Stage, SubsystemSender,
+	jaeger,
 	messages::{
 		AllMessages, AvailabilityDistributionMessage, AvailabilityStoreMessage,
 		CandidateBackingMessage, CandidateSelectionMessage, CandidateValidationMessage,
@@ -50,8 +51,8 @@ use polkadot_node_subsystem_util::{
 	request_validators,
 	request_from_runtime,
 	Validator,
-	delegated_subsystem,
 	FromJobCommand,
+	JobSender,
 	metrics::{self, prometheus},
 };
 use statement_table::{
@@ -68,8 +69,9 @@ use thiserror::Error;
 
 const LOG_TARGET: &str = "parachain::candidate-backing";
 
+/// Errors that can occur in candidate backing.
 #[derive(Debug, Error)]
-enum Error {
+pub enum Error {
 	#[error("Candidate is not found")]
 	CandidateNotFound,
 	#[error("Signature is invalid")]
@@ -142,11 +144,9 @@ impl ValidatedCandidateCommand {
 }
 
 /// Holds all data needed for candidate backing job operation.
-struct CandidateBackingJob {
+pub struct CandidateBackingJob {
 	/// The hash of the relay parent on top of which this job is doing it's work.
 	parent: Hash,
-	/// Outbound message channel sending part.
-	tx_from: mpsc::Sender<FromJobCommand>,
 	/// The `ParaId` assigned to this validator
 	assignment: Option<ParaId>,
 	/// The collator required to author the candidate, if any.
@@ -294,23 +294,20 @@ fn table_attested_to_backed(
 }
 
 async fn store_available_data(
-	tx_from: &mut mpsc::Sender<FromJobCommand>,
+	sender: &mut JobSender<impl SubsystemSender>,
 	id: Option<ValidatorIndex>,
 	n_validators: u32,
 	candidate_hash: CandidateHash,
 	available_data: AvailableData,
 ) -> Result<(), Error> {
 	let (tx, rx) = oneshot::channel();
-	tx_from.send(AllMessages::AvailabilityStore(
-			AvailabilityStoreMessage::StoreAvailableData(
-				candidate_hash,
-				id,
-				n_validators,
-				available_data,
-				tx,
-			)
-		).into()
-	).await?;
+	sender.send_message(AvailabilityStoreMessage::StoreAvailableData(
+		candidate_hash,
+		id,
+		n_validators,
+		available_data,
+		tx,
+	).into()).await;
 
 	let _ = rx.await.map_err(Error::StoreAvailableData)?;
 
@@ -321,9 +318,9 @@ async fn store_available_data(
 //
 // This will compute the erasure root internally and compare it to the expected erasure root.
 // This returns `Err()` iff there is an internal error. Otherwise, it returns either `Ok(Ok(()))` or `Ok(Err(_))`.
-#[tracing::instrument(level = "trace", skip(tx_from, pov, span), fields(subsystem = LOG_TARGET))]
+#[tracing::instrument(level = "trace", skip(sender, pov, span), fields(subsystem = LOG_TARGET))]
 async fn make_pov_available(
-	tx_from: &mut mpsc::Sender<FromJobCommand>,
+	sender: &mut JobSender<impl SubsystemSender>,
 	validator_index: Option<ValidatorIndex>,
 	n_validators: usize,
 	pov: Arc<PoV>,
@@ -361,7 +358,7 @@ async fn make_pov_available(
 		);
 
 		store_available_data(
-			tx_from,
+			sender,
 			validator_index,
 			n_validators as u32,
 			candidate_hash,
@@ -373,7 +370,7 @@ async fn make_pov_available(
 }
 
 async fn request_pov(
-	tx_from: &mut mpsc::Sender<FromJobCommand>,
+	sender: &mut JobSender<impl SubsystemSender>,
 	relay_parent: Hash,
 	from_validator: ValidatorIndex,
 	candidate_hash: CandidateHash,
@@ -381,35 +378,33 @@ async fn request_pov(
 ) -> Result<Arc<PoV>, Error> {
 
 	let (tx, rx) = oneshot::channel();
-	tx_from.send(FromJobCommand::SendMessage(AllMessages::AvailabilityDistribution(
-		AvailabilityDistributionMessage::FetchPoV {
-			relay_parent,
-			from_validator,
-			candidate_hash,
-			pov_hash,
-			tx,
-		}
-	))).await?;
+	sender.send_message(AvailabilityDistributionMessage::FetchPoV {
+		relay_parent,
+		from_validator,
+		candidate_hash,
+		pov_hash,
+		tx,
+	}.into()).await;
 
 	let pov = rx.await.map_err(|_| Error::FetchPoV)?;
 	Ok(Arc::new(pov))
 }
 
 async fn request_candidate_validation(
-	tx_from: &mut mpsc::Sender<FromJobCommand>,
+	sender: &mut JobSender<impl SubsystemSender>,
 	candidate: CandidateDescriptor,
 	pov: Arc<PoV>,
 ) -> Result<ValidationResult, Error> {
 	let (tx, rx) = oneshot::channel();
 
-	tx_from.send(AllMessages::CandidateValidation(
+	sender.send_message(AllMessages::CandidateValidation(
 			CandidateValidationMessage::ValidateFromChainState(
 				candidate,
 				pov,
 				tx,
 			)
 		).into()
-	).await?;
+	).await;
 
 	match rx.await {
 		Ok(Ok(validation_result)) => Ok(validation_result),
@@ -420,8 +415,8 @@ async fn request_candidate_validation(
 
 type BackgroundValidationResult = Result<(CandidateReceipt, CandidateCommitments, Arc<PoV>), CandidateReceipt>;
 
-struct BackgroundValidationParams<F> {
-	tx_from: mpsc::Sender<FromJobCommand>,
+struct BackgroundValidationParams<S, F> {
+	sender: JobSender<S>,
 	tx_command: mpsc::Sender<ValidatedCandidateCommand>,
 	candidate: CandidateReceipt,
 	relay_parent: Hash,
@@ -433,10 +428,13 @@ struct BackgroundValidationParams<F> {
 }
 
 async fn validate_and_make_available(
-	params: BackgroundValidationParams<impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Sync>,
+	params: BackgroundValidationParams<
+		impl SubsystemSender,
+		impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Sync,
+	>
 ) -> Result<(), Error> {
 	let BackgroundValidationParams {
-		mut tx_from,
+		mut sender,
 		mut tx_command,
 		candidate,
 		relay_parent,
@@ -456,12 +454,12 @@ async fn validate_and_make_available(
 		} => {
 			let _span = span.as_ref().map(|s| s.child("request-pov"));
 			match request_pov(
-					&mut tx_from,
-					relay_parent,
-					from_validator,
-					candidate_hash,
-					pov_hash,
-				).await {
+				&mut sender,
+				relay_parent,
+				from_validator,
+				candidate_hash,
+				pov_hash,
+			).await {
 				Err(Error::FetchPoV) => {
 					tx_command.send(ValidatedCandidateCommand::AttestNoPoV(candidate.hash())).await.map_err(Error::Mpsc)?;
 					return Ok(())
@@ -478,7 +476,7 @@ async fn validate_and_make_available(
 				.with_pov(&pov)
 				.with_para_id(candidate.descriptor().para_id)
 		});
-		request_candidate_validation(&mut tx_from, candidate.descriptor.clone(), pov.clone()).await?
+		request_candidate_validation(&mut sender, candidate.descriptor.clone(), pov.clone()).await?
 	};
 
 	let expected_commitments_hash = candidate.commitments_hash;
@@ -502,7 +500,7 @@ async fn validate_and_make_available(
 				Err(candidate)
 			} else {
 				let erasure_valid = make_pov_available(
-					&mut tx_from,
+					&mut sender,
 					validator_index,
 					n_validators,
 					pov.clone(),
@@ -544,6 +542,7 @@ impl CandidateBackingJob {
 	/// Run asynchronously.
 	async fn run_loop(
 		mut self,
+		mut sender: JobSender<impl SubsystemSender>,
 		mut rx_to: mpsc::Receiver<CandidateBackingMessage>,
 		span: PerLeafSpan,
 	) -> Result<(), Error> {
@@ -552,7 +551,7 @@ impl CandidateBackingJob {
 				validated_command = self.background_validation.next() => {
 					let _span = span.child("process-validation-result");
 					if let Some(c) = validated_command {
-						self.handle_validated_candidate_command(&span, c).await?;
+						self.handle_validated_candidate_command(&span, &mut sender, c).await?;
 					} else {
 						panic!("`self` hasn't dropped and `self` holds a reference to this sender; qed");
 					}
@@ -563,7 +562,7 @@ impl CandidateBackingJob {
 						// we intentionally want spans created in `process_msg` to descend from the
 						// `span ` which is longer-lived than this ephemeral timing span.
 						let _timing_span = span.child("process-message");
-						self.process_msg(&span, msg).await?;
+						self.process_msg(&span, &mut sender, msg).await?;
 					}
 				}
 			}
@@ -572,10 +571,11 @@ impl CandidateBackingJob {
 		Ok(())
 	}
 
-	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
+	#[tracing::instrument(level = "trace", skip(self, parent_span, sender), fields(subsystem = LOG_TARGET))]
 	async fn handle_validated_candidate_command(
 		&mut self,
 		parent_span: &jaeger::Span,
+		sender: &mut JobSender<impl SubsystemSender>,
 		command: ValidatedCandidateCommand,
 	) -> Result<(), Error> {
 		let candidate_hash = command.candidate_hash();
@@ -596,15 +596,20 @@ impl CandidateBackingJob {
 								commitments,
 							});
 							if let Some(stmt) = self.sign_import_and_distribute_statement(
+								sender,
 								statement,
 								parent_span,
 							).await? {
-								self.issue_candidate_seconded_message(stmt).await?;
+								sender.send_message(
+									CandidateSelectionMessage::Seconded(self.parent, stmt).into()
+								).await;
 							}
 						}
 					}
 					Err(candidate) => {
-						self.issue_candidate_invalid_message(candidate).await?;
+						sender.send_message(
+							CandidateSelectionMessage::Invalid(self.parent, candidate).into()
+						).await;
 					}
 				}
 			}
@@ -615,7 +620,7 @@ impl CandidateBackingJob {
 				if !self.issued_statements.contains(&candidate_hash) {
 					if res.is_ok() {
 						let statement = Statement::Valid(candidate_hash);
-						self.sign_import_and_distribute_statement(statement, &parent_span).await?;
+						self.sign_import_and_distribute_statement(sender, statement, &parent_span).await?;
 					}
 					self.issued_statements.insert(candidate_hash);
 				}
@@ -627,7 +632,7 @@ impl CandidateBackingJob {
 						// Ok, another try:
 						let c_span = span.as_ref().map(|s| s.child("try"));
 						let attesting = attesting.clone();
-						self.kick_off_validation_work(attesting, c_span).await?
+						self.kick_off_validation_work(sender, attesting, c_span).await?
 					}
 
 				} else {
@@ -643,10 +648,12 @@ impl CandidateBackingJob {
 		Ok(())
 	}
 
-	#[tracing::instrument(level = "trace", skip(self, params), fields(subsystem = LOG_TARGET))]
+	#[tracing::instrument(level = "trace", skip(self, sender, params), fields(subsystem = LOG_TARGET))]
 	async fn background_validate_and_make_available(
 		&mut self,
+		sender: &mut JobSender<impl SubsystemSender>,
 		params: BackgroundValidationParams<
+			impl SubsystemSender,
 			impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Send + 'static + Sync
 		>,
 	) -> Result<(), Error> {
@@ -658,36 +665,19 @@ impl CandidateBackingJob {
 					tracing::error!(target: LOG_TARGET, "Failed to validate and make available: {:?}", e);
 				}
 			};
-			self.tx_from.send(FromJobCommand::Spawn("Backing Validation", bg.boxed())).await?;
+			sender.send_command(FromJobCommand::Spawn("Backing Validation", bg.boxed())).await?;
 		}
 
 		Ok(())
 	}
 
-	async fn issue_candidate_invalid_message(
-		&mut self,
-		candidate: CandidateReceipt,
-	) -> Result<(), Error> {
-		self.tx_from.send(AllMessages::from(CandidateSelectionMessage::Invalid(self.parent, candidate)).into()).await?;
-
-		Ok(())
-	}
-
-	async fn issue_candidate_seconded_message(
-		&mut self,
-		statement: SignedFullStatement,
-	) -> Result<(), Error> {
-		self.tx_from.send(AllMessages::from(CandidateSelectionMessage::Seconded(self.parent, statement)).into()).await?;
-
-		Ok(())
-	}
-
 	/// Kick off background validation with intent to second.
-	#[tracing::instrument(level = "trace", skip(self, parent_span, pov), fields(subsystem = LOG_TARGET))]
+	#[tracing::instrument(level = "trace", skip(self, parent_span, sender, pov), fields(subsystem = LOG_TARGET))]
 	async fn validate_and_second(
 		&mut self,
 		parent_span: &jaeger::Span,
 		root_span: &jaeger::Span,
+		sender: &mut JobSender<impl SubsystemSender>,
 		candidate: &CandidateReceipt,
 		pov: Arc<PoV>,
 	) -> Result<(), Error> {
@@ -695,7 +685,9 @@ impl CandidateBackingJob {
 		if self.required_collator.as_ref()
 			.map_or(false, |c| c != &candidate.descriptor().collator)
 		{
-			self.issue_candidate_invalid_message(candidate.clone()).await?;
+			sender.send_message(
+				CandidateSelectionMessage::Invalid(self.parent, candidate.clone()).into()
+			).await;
 			return Ok(());
 		}
 
@@ -715,29 +707,36 @@ impl CandidateBackingJob {
 			"Validate and second candidate",
 		);
 
-		self.background_validate_and_make_available(BackgroundValidationParams {
-			tx_from: self.tx_from.clone(),
-			tx_command: self.background_validation_tx.clone(),
-			candidate: candidate.clone(),
-			relay_parent: self.parent,
-			pov: PoVData::Ready(pov),
-			validator_index: self.table_context.validator.as_ref().map(|v| v.index()),
-			n_validators: self.table_context.validators.len(),
-			span,
-			make_command: ValidatedCandidateCommand::Second,
-		}).await?;
+		let bg_sender = sender.clone();
+		self.background_validate_and_make_available(
+			sender,
+			BackgroundValidationParams {
+				sender: bg_sender,
+				tx_command: self.background_validation_tx.clone(),
+				candidate: candidate.clone(),
+				relay_parent: self.parent,
+				pov: PoVData::Ready(pov),
+				validator_index: self.table_context.validator.as_ref().map(|v| v.index()),
+				n_validators: self.table_context.validators.len(),
+				span,
+				make_command: ValidatedCandidateCommand::Second,
+			}
+		).await?;
 
 		Ok(())
 	}
 
 	async fn sign_import_and_distribute_statement(
 		&mut self,
+		sender: &mut JobSender<impl SubsystemSender>,
 		statement: Statement,
 		parent_span: &jaeger::Span,
 	) -> Result<Option<SignedFullStatement>, Error> {
 		if let Some(signed_statement) = self.sign_statement(statement).await {
-			self.import_statement(&signed_statement, parent_span).await?;
-			self.distribute_signed_statement(signed_statement.clone()).await?;
+			self.import_statement(sender, &signed_statement, parent_span).await?;
+			let smsg = StatementDistributionMessage::Share(self.parent, signed_statement.clone());
+			sender.send_unbounded_message(smsg.into());
+
 			Ok(Some(signed_statement))
 		} else {
 			Ok(None)
@@ -745,26 +744,25 @@ impl CandidateBackingJob {
 	}
 
 	/// Check if there have happened any new misbehaviors and issue necessary messages.
-	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
-	async fn issue_new_misbehaviors(&mut self) -> Result<(), Error> {
+	#[tracing::instrument(level = "trace", skip(self, sender), fields(subsystem = LOG_TARGET))]
+	async fn issue_new_misbehaviors(&mut self, sender: &mut JobSender<impl SubsystemSender>) {
 		// collect the misbehaviors to avoid double mutable self borrow issues
 		let misbehaviors: Vec<_> = self.table.drain_misbehaviors().collect();
 		for (validator_id, report) in misbehaviors {
-			self.send_to_provisioner(
+			sender.send_message(
 				ProvisionerMessage::ProvisionableData(
 					self.parent,
 					ProvisionableData::MisbehaviorReport(self.parent, validator_id, report)
-				)
-			).await?
+				).into()
+			).await;
 		}
-
-		Ok(())
 	}
 
 	/// Import a statement into the statement table and return the summary of the import.
-	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
+	#[tracing::instrument(level = "trace", skip(self, sender), fields(subsystem = LOG_TARGET))]
 	async fn import_statement(
 		&mut self,
+		sender: &mut JobSender<impl SubsystemSender>,
 		statement: &SignedFullStatement,
 		parent_span: &jaeger::Span,
 	) -> Result<Option<TableSummary>, Error> {
@@ -807,7 +805,7 @@ impl CandidateBackingJob {
 						self.parent,
 						ProvisionableData::BackedCandidate(backed.receipt()),
 					);
-					self.send_to_provisioner(message).await?;
+					sender.send_message(message.into()).await;
 
 					span.as_ref().map(|s| s.child("backed"));
 					span
@@ -821,7 +819,7 @@ impl CandidateBackingJob {
 			None
 		};
 
-		self.issue_new_misbehaviors().await?;
+		self.issue_new_misbehaviors(sender).await;
 
 		// It is important that the child span is dropped before its parent span (`unbacked_span`)
 		drop(import_statement_span);
@@ -830,8 +828,13 @@ impl CandidateBackingJob {
 		Ok(summary)
 	}
 
-	#[tracing::instrument(level = "trace", skip(self, root_span), fields(subsystem = LOG_TARGET))]
-	async fn process_msg(&mut self, root_span: &jaeger::Span, msg: CandidateBackingMessage) -> Result<(), Error> {
+	#[tracing::instrument(level = "trace", skip(self, root_span, sender), fields(subsystem = LOG_TARGET))]
+	async fn process_msg(
+		&mut self,
+		root_span: &jaeger::Span,
+		sender: &mut JobSender<impl SubsystemSender>,
+		msg: CandidateBackingMessage,
+	) -> Result<(), Error> {
 		match msg {
 			CandidateBackingMessage::Second(relay_parent, candidate, pov) => {
 				let _timer = self.metrics.time_process_second();
@@ -856,7 +859,7 @@ impl CandidateBackingJob {
 
 					if !self.issued_statements.contains(&candidate_hash) {
 						let pov = Arc::new(pov);
-						self.validate_and_second(&span, &root_span, &candidate, pov).await?;
+						self.validate_and_second(&span, &root_span, sender, &candidate, pov).await?;
 					}
 				}
 			}
@@ -868,7 +871,7 @@ impl CandidateBackingJob {
 					.with_relay_parent(_relay_parent);
 
 				self.check_statement_signature(&statement)?;
-				match self.maybe_validate_and_import(&span, &root_span, statement).await {
+				match self.maybe_validate_and_import(&span, &root_span, sender, statement).await {
 					Err(Error::ValidationFailed(_)) => return Ok(()),
 					Err(e) => return Err(e),
 					Ok(()) => (),
@@ -893,9 +896,10 @@ impl CandidateBackingJob {
 	}
 
 	/// Kick off validation work and distribute the result as a signed statement.
-	#[tracing::instrument(level = "trace", skip(self, attesting, span), fields(subsystem = LOG_TARGET))]
+	#[tracing::instrument(level = "trace", skip(self, sender, attesting, span), fields(subsystem = LOG_TARGET))]
 	async fn kick_off_validation_work(
 		&mut self,
+		sender: &mut JobSender<impl SubsystemSender>,
 		attesting: AttestingData,
 		span: Option<jaeger::Span>,
 	) -> Result<(), Error> {
@@ -925,34 +929,38 @@ impl CandidateBackingJob {
 			return Ok(());
 		}
 
+		let bg_sender = sender.clone();
 		let pov = PoVData::FetchFromValidator {
 			from_validator: attesting.from_validator,
 			candidate_hash,
 			pov_hash: attesting.pov_hash,
 		};
-
-		self.background_validate_and_make_available(BackgroundValidationParams {
-			tx_from: self.tx_from.clone(),
-			tx_command: self.background_validation_tx.clone(),
-			candidate: attesting.candidate,
-			relay_parent: self.parent,
-			pov,
-			validator_index: self.table_context.validator.as_ref().map(|v| v.index()),
-			n_validators: self.table_context.validators.len(),
-			span,
-			make_command: ValidatedCandidateCommand::Attest,
-		}).await
+		self.background_validate_and_make_available(
+			sender,
+			BackgroundValidationParams {
+				sender: bg_sender,
+				tx_command: self.background_validation_tx.clone(),
+				candidate: attesting.candidate,
+				relay_parent: self.parent,
+				pov,
+				validator_index: self.table_context.validator.as_ref().map(|v| v.index()),
+				n_validators: self.table_context.validators.len(),
+				span,
+				make_command: ValidatedCandidateCommand::Attest,
+			},
+		).await
 	}
 
 	/// Import the statement and kick off validation work if it is a part of our assignment.
-	#[tracing::instrument(level = "trace", skip(self, parent_span), fields(subsystem = LOG_TARGET))]
+	#[tracing::instrument(level = "trace", skip(self, parent_span, root_span, sender), fields(subsystem = LOG_TARGET))]
 	async fn maybe_validate_and_import(
 		&mut self,
 		parent_span: &jaeger::Span,
 		root_span: &jaeger::Span,
+		sender: &mut JobSender<impl SubsystemSender>,
 		statement: SignedFullStatement,
 	) -> Result<(), Error> {
-		if let Some(summary) = self.import_statement(&statement, parent_span).await? {
+		if let Some(summary) = self.import_statement(sender, &statement, parent_span).await? {
 			if Some(summary.group_id) != self.assignment {
 				return Ok(())
 			}
@@ -989,7 +997,7 @@ impl CandidateBackingJob {
 							attesting.backing.push(statement.validator_index());
 							return Ok(())
 						} else {
-							// No job, so start another try with current validator:
+							// No job, so start another with current validator:
 							attesting.from_validator = statement.validator_index();
 							(attesting.clone(), span.as_ref().map(|s| s.child("try")))
 						}
@@ -1000,6 +1008,7 @@ impl CandidateBackingJob {
 			};
 
 			self.kick_off_validation_work(
+				sender,
 				attesting,
 				span,
 			).await?;
@@ -1089,20 +1098,6 @@ impl CandidateBackingJob {
 	fn remove_unbacked_span(&mut self, hash: &CandidateHash) -> Option<jaeger::Span> {
 		self.unbacked_candidates.remove(hash)
 	}
-
-	async fn send_to_provisioner(&mut self, msg: ProvisionerMessage) -> Result<(), Error> {
-		self.tx_from.send(AllMessages::from(msg).into()).await?;
-
-		Ok(())
-	}
-
-	async fn distribute_signed_statement(&mut self, s: SignedFullStatement) -> Result<(), Error> {
-		let smsg = StatementDistributionMessage::Share(self.parent, s);
-
-		self.tx_from.send(AllMessages::from(smsg).into()).await?;
-
-		Ok(())
-	}
 }
 
 impl util::JobTrait for CandidateBackingJob {
@@ -1113,14 +1108,14 @@ impl util::JobTrait for CandidateBackingJob {
 
 	const NAME: &'static str = "CandidateBackingJob";
 
-	#[tracing::instrument(skip(span, keystore, metrics, rx_to, tx_from), fields(subsystem = LOG_TARGET))]
-	fn run(
+	#[tracing::instrument(skip(span, keystore, metrics, rx_to, sender), fields(subsystem = LOG_TARGET))]
+	fn run<S: SubsystemSender>(
 		parent: Hash,
 		span: Arc<jaeger::Span>,
 		keystore: SyncCryptoStorePtr,
 		metrics: Metrics,
 		rx_to: mpsc::Receiver<Self::ToJob>,
-		mut tx_from: mpsc::Sender<FromJobCommand>,
+		mut sender: JobSender<S>,
 	) -> Pin<Box<dyn Future<Output = Result<(), Self::Error>> + Send>> {
 		async move {
 			macro_rules! try_runtime_api {
@@ -1147,14 +1142,14 @@ impl util::JobTrait for CandidateBackingJob {
 			let _span = span.child("runtime-apis");
 
 			let (validators, groups, session_index, cores) = futures::try_join!(
-				try_runtime_api!(request_validators(parent, &mut tx_from).await),
-				try_runtime_api!(request_validator_groups(parent, &mut tx_from).await),
-				try_runtime_api!(request_session_index_for_child(parent, &mut tx_from).await),
-				try_runtime_api!(request_from_runtime(
+				request_validators(parent, &mut sender).await,
+				request_validator_groups(parent, &mut sender).await,
+				request_session_index_for_child(parent, &mut sender).await,
+				request_from_runtime(
 					parent,
-					&mut tx_from,
+					&mut sender,
 					|tx| RuntimeApiRequest::AvailabilityCores(tx),
-				).await),
+				).await,
 			).map_err(Error::JoinMultiple)?;
 
 			let validators = try_runtime_api!(validators);
@@ -1231,7 +1226,6 @@ impl util::JobTrait for CandidateBackingJob {
 			let (background_tx, background_rx) = mpsc::channel(16);
 			let job = CandidateBackingJob {
 				parent,
-				tx_from,
 				assignment,
 				required_collator,
 				issued_statements: HashSet::new(),
@@ -1249,7 +1243,7 @@ impl util::JobTrait for CandidateBackingJob {
 			};
 			drop(_span);
 
-			job.run_loop(rx_to, span).await
+			job.run_loop(sender, rx_to, span).await
 		}.boxed()
 	}
 }
@@ -1345,7 +1339,9 @@ impl metrics::Metrics for Metrics {
 	}
 }
 
-delegated_subsystem!(CandidateBackingJob(SyncCryptoStorePtr, Metrics) <- CandidateBackingMessage as CandidateBackingSubsystem);
+/// The candidate backing subsystem.
+pub type CandidateBackingSubsystem<Spawner>
+	= polkadot_node_subsystem_util::JobSubsystem<CandidateBackingJob, Spawner>;
 
 #[cfg(test)]
 mod tests {
@@ -1363,6 +1359,7 @@ mod tests {
 	use sp_keystore::{CryptoStore, SyncCryptoStore};
 	use statement_table::v1::Misbehavior;
 	use std::collections::HashMap;
+	use sp_tracing as _;
 
 	fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec<ValidatorId> {
 		val_ids.iter().map(|v| v.public().into()).collect()
@@ -1479,7 +1476,11 @@ mod tests {
 
 		let (context, virtual_overseer) = polkadot_node_subsystem_test_helpers::make_subsystem_context(pool.clone());
 
-		let subsystem = CandidateBackingSubsystem::run(context, keystore, Metrics(None), pool.clone());
+		let subsystem = CandidateBackingSubsystem::new(
+			pool.clone(),
+			keystore,
+			Metrics(None),
+		).run(context);
 
 		let test_fut = test(TestHarness {
 			virtual_overseer,
@@ -2670,7 +2671,7 @@ mod tests {
 	// Test whether we retry on failed PoV fetching.
 	#[test]
 	fn retry_works() {
-		sp_tracing::try_init_simple();
+		// sp_tracing::try_init_simple();
 		let test_state = TestState::default();
 		test_harness(test_state.keystore.clone(), |test_harness| async move {
 			let TestHarness { mut virtual_overseer } = test_harness;

polkadot/node/core/bitfield-signing/Cargo.toml

@@ -13,3 +13,6 @@ polkadot-node-subsystem-util = { path = "../../subsystem-util" }
 sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" }
 wasm-timer = "0.2.5"
 thiserror = "1.0.23"
+
+[dev-dependencies]
+polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }

polkadot/node/core/bitfield-signing/src/lib.rs

@@ -23,15 +23,16 @@
 use futures::{channel::{mpsc, oneshot}, lock::Mutex, prelude::*, future, Future};
 use sp_keystore::{Error as KeystoreError, SyncCryptoStorePtr};
 use polkadot_node_subsystem::{
-	jaeger, PerLeafSpan,
+	jaeger, PerLeafSpan, SubsystemSender,
 	messages::{
-		AllMessages, AvailabilityStoreMessage, BitfieldDistributionMessage,
+		AvailabilityStoreMessage, BitfieldDistributionMessage,
 		BitfieldSigningMessage, RuntimeApiMessage, RuntimeApiRequest,
 	},
 	errors::RuntimeApiError,
 };
 use polkadot_node_subsystem_util::{
-	self as util, JobManager, JobTrait, Validator, FromJobCommand, metrics::{self, prometheus},
+	self as util, JobSubsystem, JobTrait, Validator, metrics::{self, prometheus},
+	JobSender,
 };
 use polkadot_primitives::v1::{AvailabilityBitfield, CoreState, Hash, ValidatorIndex};
 use std::{pin::Pin, time::Duration, iter::FromIterator, sync::Arc};
@@ -73,7 +74,7 @@ pub enum Error {
 async fn get_core_availability(
 	core: &CoreState,
 	validator_idx: ValidatorIndex,
-	sender: &Mutex<&mut mpsc::Sender<FromJobCommand>>,
+	sender: &Mutex<&mut impl SubsystemSender>,
 	span: &jaeger::Span,
 ) -> Result<bool, Error> {
 	if let &CoreState::Occupied(ref core) = core {
@@ -83,14 +84,14 @@ async fn get_core_availability(
 		sender
 			.lock()
 			.await
-			.send(
-				AllMessages::from(AvailabilityStoreMessage::QueryChunkAvailability(
+			.send_message(
+				AvailabilityStoreMessage::QueryChunkAvailability(
 					core.candidate_hash,
 					validator_idx,
 					tx,
-				)).into(),
+				).into(),
 			)
-			.await?;
+			.await;
 
 		let res = rx.await.map_err(Into::into);
 
@@ -111,12 +112,15 @@ async fn get_core_availability(
 /// delegates to the v1 runtime API
 async fn get_availability_cores(
 	relay_parent: Hash,
-	sender: &mut mpsc::Sender<FromJobCommand>,
+	sender: &mut impl SubsystemSender,
 ) -> Result<Vec<CoreState>, Error> {
 	let (tx, rx) = oneshot::channel();
 	sender
-		.send(AllMessages::from(RuntimeApiMessage::Request(relay_parent, RuntimeApiRequest::AvailabilityCores(tx))).into())
-		.await?;
+		.send_message(RuntimeApiMessage::Request(
+			relay_parent,
+			RuntimeApiRequest::AvailabilityCores(tx),
+		).into())
+		.await;
 	match rx.await {
 		Ok(Ok(out)) => Ok(out),
 		Ok(Err(runtime_err)) => Err(runtime_err.into()),
@@ -133,7 +137,7 @@ async fn construct_availability_bitfield(
 	relay_parent: Hash,
 	span: &jaeger::Span,
 	validator_idx: ValidatorIndex,
-	sender: &mut mpsc::Sender<FromJobCommand>,
+	sender: &mut impl SubsystemSender,
 ) -> Result<AvailabilityBitfield, Error> {
 	// get the set of availability cores from the runtime
 	let availability_cores = {
@@ -223,13 +227,13 @@ impl JobTrait for BitfieldSigningJob {
 
 	/// Run a job for the parent block indicated
 	#[tracing::instrument(skip(span, keystore, metrics, _receiver, sender), fields(subsystem = LOG_TARGET))]
-	fn run(
+	fn run<S: SubsystemSender>(
 		relay_parent: Hash,
 		span: Arc<jaeger::Span>,
 		keystore: Self::RunArgs,
 		metrics: Self::Metrics,
 		_receiver: mpsc::Receiver<BitfieldSigningMessage>,
-		mut sender: mpsc::Sender<FromJobCommand>,
+		mut sender: JobSender<S>,
 	) -> Pin<Box<dyn Future<Output = Result<(), Self::Error>> + Send>> {
 		let metrics = metrics.clone();
 		async move {
@@ -239,7 +243,7 @@ impl JobTrait for BitfieldSigningJob {
 
 			// now do all the work we can before we need to wait for the availability store
 			// if we're not a validator, we can just succeed effortlessly
-			let validator = match Validator::new(relay_parent, keystore.clone(), sender.clone()).await {
+			let validator = match Validator::new(relay_parent, keystore.clone(), &mut sender).await {
 				Ok(validator) => validator,
 				Err(util::Error::NotAValidator) => return Ok(()),
 				Err(err) => return Err(Error::Util(err)),
@@ -260,7 +264,7 @@ impl JobTrait for BitfieldSigningJob {
 					relay_parent,
 					&span_availability,
 					validator.index(),
-					&mut sender,
+					sender.subsystem_sender(),
 				).await
 			{
 				Err(Error::Runtime(runtime_err)) => {
@@ -295,26 +299,27 @@ impl JobTrait for BitfieldSigningJob {
 			let _span = span.child("gossip");
 
 			sender
-				.send(
-					AllMessages::from(
-						BitfieldDistributionMessage::DistributeBitfield(relay_parent, signed_bitfield),
-					).into(),
-				)
-				.await
-				.map_err(Into::into)
+				.send_message(BitfieldDistributionMessage::DistributeBitfield(
+					relay_parent,
+					signed_bitfield,
+				).into())
+				.await;
+
+			Ok(())
 		}
 		.boxed()
 	}
 }
 
 /// BitfieldSigningSubsystem manages a number of bitfield signing jobs.
-pub type BitfieldSigningSubsystem<Spawner, Context> = JobManager<Spawner, Context, BitfieldSigningJob>;
+pub type BitfieldSigningSubsystem<Spawner> = JobSubsystem<BitfieldSigningJob, Spawner>;
 
 #[cfg(test)]
 mod tests {
 	use super::*;
 	use futures::{pin_mut, executor::block_on};
 	use polkadot_primitives::v1::{CandidateHash, OccupiedCore};
+	use polkadot_node_subsystem::messages::AllMessages;
 
 	fn occupied_core(para_id: u32, candidate_hash: CandidateHash) -> CoreState {
 		CoreState::Occupied(OccupiedCore {
@@ -332,10 +337,10 @@ mod tests {
 	#[test]
 	fn construct_availability_bitfield_works() {
 		block_on(async move {
-			let (mut sender, mut receiver) = mpsc::channel(10);
 			let relay_parent = Hash::default();
 			let validator_index = ValidatorIndex(1u32);
 
+			let (mut sender, mut receiver) = polkadot_node_subsystem_test_helpers::sender_receiver();
 			let future = construct_availability_bitfield(
 				relay_parent,
 				&jaeger::Span::Disabled,
@@ -350,18 +355,14 @@ mod tests {
 			loop {
 				futures::select! {
 					m = receiver.next() => match m.unwrap() {
-						FromJobCommand::SendMessage(
-							AllMessages::RuntimeApi(
-								RuntimeApiMessage::Request(rp, RuntimeApiRequest::AvailabilityCores(tx)),
-							),
+						AllMessages::RuntimeApi(
+							RuntimeApiMessage::Request(rp, RuntimeApiRequest::AvailabilityCores(tx)),
 						) => {
 							assert_eq!(relay_parent, rp);
 							tx.send(Ok(vec![CoreState::Free, occupied_core(1, hash_a), occupied_core(2, hash_b)])).unwrap();
-						},
-						FromJobCommand::SendMessage(
-							AllMessages::AvailabilityStore(
-								AvailabilityStoreMessage::QueryChunkAvailability(c_hash, vidx, tx),
-							),
+						}
+						AllMessages::AvailabilityStore(
+							AvailabilityStoreMessage::QueryChunkAvailability(c_hash, vidx, tx),
 						) => {
 							assert_eq!(validator_index, vidx);
 

polkadot/node/core/candidate-selection/Cargo.toml

@@ -18,3 +18,4 @@ polkadot-node-subsystem-util = { path = "../../subsystem-util" }
 
 [dev-dependencies]
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
+polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }

polkadot/node/core/candidate-selection/src/lib.rs

@@ -25,16 +25,16 @@ use futures::{
 };
 use sp_keystore::SyncCryptoStorePtr;
 use polkadot_node_subsystem::{
-	jaeger, PerLeafSpan,
+	jaeger, PerLeafSpan, SubsystemSender,
 	errors::ChainApiError,
 	messages::{
-		AllMessages, CandidateBackingMessage, CandidateSelectionMessage, CollatorProtocolMessage,
+		CandidateBackingMessage, CandidateSelectionMessage, CollatorProtocolMessage,
 		RuntimeApiRequest,
 	},
 };
 use polkadot_node_subsystem_util::{
-	self as util, request_from_runtime, request_validator_groups, delegated_subsystem,
-	JobTrait, FromJobCommand, Validator, metrics::{self, prometheus},
+	self as util, request_from_runtime, request_validator_groups, JobSubsystem,
+	JobTrait, JobSender, Validator, metrics::{self, prometheus},
 };
 use polkadot_primitives::v1::{
 	CandidateReceipt, CollatorId, CoreState, CoreIndex, Hash, Id as ParaId, PoV, BlockNumber,
@@ -45,22 +45,24 @@ use thiserror::Error;
 
 const LOG_TARGET: &'static str = "parachain::candidate-selection";
 
-struct CandidateSelectionJob {
+/// A per-block job in the candidate selection subsystem.
+pub struct CandidateSelectionJob {
 	assignment: ParaId,
-	sender: mpsc::Sender<FromJobCommand>,
 	receiver: mpsc::Receiver<CandidateSelectionMessage>,
 	metrics: Metrics,
 	seconded_candidate: Option<CollatorId>,
 }
 
+/// Errors in the candidate selection subsystem.
 #[derive(Debug, Error)]
-enum Error {
-	#[error(transparent)]
-	Sending(#[from] mpsc::SendError),
+pub enum Error {
+	/// An error in utilities.
 	#[error(transparent)]
 	Util(#[from] util::Error),
+	/// An error receiving on a oneshot channel.
 	#[error(transparent)]
 	OneshotRecv(#[from] oneshot::Canceled),
+	/// An error interacting with the chain API.
 	#[error(transparent)]
 	ChainApi(#[from] ChainApiError),
 }
@@ -94,13 +96,13 @@ impl JobTrait for CandidateSelectionJob {
 	const NAME: &'static str = "CandidateSelectionJob";
 
 	#[tracing::instrument(skip(keystore, metrics, receiver, sender), fields(subsystem = LOG_TARGET))]
-	fn run(
+	fn run<S: SubsystemSender>(
 		relay_parent: Hash,
 		span: Arc<jaeger::Span>,
 		keystore: Self::RunArgs,
 		metrics: Self::Metrics,
 		receiver: mpsc::Receiver<CandidateSelectionMessage>,
-		mut sender: mpsc::Sender<FromJobCommand>,
+		mut sender: JobSender<S>,
 	) -> Pin<Box<dyn Future<Output = Result<(), Self::Error>> + Send>> {
 		let span = PerLeafSpan::new(span, "candidate-selection");
 		async move {
@@ -108,12 +110,12 @@ impl JobTrait for CandidateSelectionJob {
 				.with_relay_parent(relay_parent)
 				.with_stage(jaeger::Stage::CandidateSelection);
 			let (groups, cores) = futures::try_join!(
-				try_runtime_api!(request_validator_groups(relay_parent, &mut sender).await),
-				try_runtime_api!(request_from_runtime(
+				request_validator_groups(relay_parent, &mut sender).await,
+				request_from_runtime(
 					relay_parent,
 					&mut sender,
 					|tx| RuntimeApiRequest::AvailabilityCores(tx),
-				).await),
+				).await,
 			)?;
 
 			let (validator_groups, group_rotation_info) = try_runtime_api!(groups);
@@ -126,7 +128,7 @@ impl JobTrait for CandidateSelectionJob {
 
 			let n_cores = cores.len();
 
-			let validator = match Validator::new(relay_parent, keystore.clone(), sender.clone()).await {
+			let validator = match Validator::new(relay_parent, keystore.clone(), &mut sender).await {
 				Ok(validator) => validator,
 				Err(util::Error::NotAValidator) => return Ok(()),
 				Err(err) => return Err(Error::Util(err)),
@@ -198,20 +200,20 @@ impl JobTrait for CandidateSelectionJob {
 
 			drop(assignment_span);
 
-			CandidateSelectionJob::new(assignment, metrics, sender, receiver).run_loop(&span).await
+			CandidateSelectionJob::new(assignment, metrics, receiver)
+				.run_loop(&span, sender.subsystem_sender())
+				.await
 		}.boxed()
 	}
 }
 
 impl CandidateSelectionJob {
-	pub fn new(
+	fn new(
 		assignment: ParaId,
 		metrics: Metrics,
-		sender: mpsc::Sender<FromJobCommand>,
 		receiver: mpsc::Receiver<CandidateSelectionMessage>,
 	) -> Self {
 		Self {
-			sender,
 			receiver,
 			metrics,
 			assignment,
@@ -219,7 +221,11 @@ impl CandidateSelectionJob {
 		}
 	}
 
-	async fn run_loop(&mut self, span: &jaeger::Span) -> Result<(), Error> {
+	async fn run_loop(
+		&mut self,
+		span: &jaeger::Span,
+		sender: &mut impl SubsystemSender,
+	) -> Result<(), Error> {
 		let span = span.child("run-loop")
 			.with_stage(jaeger::Stage::CandidateSelection);
 
@@ -231,7 +237,7 @@ impl CandidateSelectionJob {
 					collator_id,
 				)) => {
 					let _span = span.child("handle-collation");
-					self.handle_collation(relay_parent, para_id, collator_id).await;
+					self.handle_collation(sender, relay_parent, para_id, collator_id).await;
 				}
 				Some(CandidateSelectionMessage::Invalid(
 					_relay_parent,
@@ -241,28 +247,26 @@ impl CandidateSelectionJob {
 						.with_stage(jaeger::Stage::CandidateSelection)
 						.with_candidate(candidate_receipt.hash())
 						.with_relay_parent(_relay_parent);
-					self.handle_invalid(candidate_receipt).await;
+					self.handle_invalid(sender, candidate_receipt).await;
 				}
 				Some(CandidateSelectionMessage::Seconded(_relay_parent, statement)) => {
 					let _span = span.child("handle-seconded")
 						.with_stage(jaeger::Stage::CandidateSelection)
 						.with_candidate(statement.payload().candidate_hash())
 						.with_relay_parent(_relay_parent);
-					self.handle_seconded(statement).await;
+					self.handle_seconded(sender, statement).await;
 				}
 				None => break,
 			}
 		}
 
-		// closing the sender here means that we don't deadlock in tests
-		self.sender.close_channel();
-
 		Ok(())
 	}
 
-	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
+	#[tracing::instrument(level = "trace", skip(self, sender), fields(subsystem = LOG_TARGET))]
 	async fn handle_collation(
 		&mut self,
+		sender: &mut impl SubsystemSender,
 		relay_parent: Hash,
 		para_id: ParaId,
 		collator_id: CollatorId,
@@ -276,13 +280,7 @@ impl CandidateSelectionJob {
 				collator_id,
 				para_id,
 			);
-			if let Err(err) = forward_invalidity_note(&collator_id, &mut self.sender).await {
-				tracing::warn!(
-					target: LOG_TARGET,
-					err = ?err,
-					"failed to forward invalidity note",
-				);
-			}
+			forward_invalidity_note(&collator_id, sender).await;
 			return;
 		}
 
@@ -292,7 +290,7 @@ impl CandidateSelectionJob {
 					relay_parent,
 					para_id,
 					collator_id.clone(),
-					self.sender.clone(),
+					sender,
 				).await {
 					Ok(response) => response,
 					Err(err) => {
@@ -305,21 +303,23 @@ impl CandidateSelectionJob {
 					}
 				};
 
-			match second_candidate(
+			second_candidate(
 				relay_parent,
 				candidate_receipt,
 				pov,
-				&mut self.sender,
+				sender,
 				&self.metrics,
-			).await {
-				Err(err) => tracing::warn!(target: LOG_TARGET, err = ?err, "failed to second a candidate"),
-				Ok(()) => self.seconded_candidate = Some(collator_id),
-			}
+			).await;
+			self.seconded_candidate = Some(collator_id);
 		}
 	}
 
-	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
-	async fn handle_invalid(&mut self, candidate_receipt: CandidateReceipt) {
+	#[tracing::instrument(level = "trace", skip(self, sender), fields(subsystem = LOG_TARGET))]
+	async fn handle_invalid(
+		&mut self,
+		sender: &mut impl SubsystemSender,
+		candidate_receipt: CandidateReceipt,
+	) {
 		let _timer = self.metrics.time_handle_invalid();
 
 		let received_from = match &self.seconded_candidate {
@@ -338,21 +338,15 @@ impl CandidateSelectionJob {
 			"received invalidity note for candidate",
 		);
 
-		let result =
-			if let Err(err) = forward_invalidity_note(received_from, &mut self.sender).await {
-				tracing::warn!(
-					target: LOG_TARGET,
-					err = ?err,
-					"failed to forward invalidity note",
-				);
-				Err(())
-			} else {
-				Ok(())
-			};
-		self.metrics.on_invalid_selection(result);
+		forward_invalidity_note(received_from, sender).await;
+		self.metrics.on_invalid_selection();
 	}
 
-	async fn handle_seconded(&mut self, statement: SignedFullStatement) {
+	async fn handle_seconded(
+		&mut self,
+		sender: &mut impl SubsystemSender,
+		statement: SignedFullStatement,
+	) {
 		let received_from = match &self.seconded_candidate {
 			Some(peer) => peer,
 			None => {
@@ -369,27 +363,13 @@ impl CandidateSelectionJob {
 			"received seconded note for candidate",
 		);
 
-		if let Err(e) = self.sender
-			.send(AllMessages::from(CollatorProtocolMessage::NoteGoodCollation(received_from.clone())).into()).await
-		{
-			tracing::debug!(
-				target: LOG_TARGET,
-				error = ?e,
-				"failed to note good collator"
-			);
-		}
+		sender
+			.send_message(CollatorProtocolMessage::NoteGoodCollation(received_from.clone()).into())
+			.await;
 
-		if let Err(e) = self.sender
-			.send(AllMessages::from(
-				CollatorProtocolMessage::NotifyCollationSeconded(received_from.clone(), statement)
-			).into()).await
-		{
-			tracing::debug!(
-				target: LOG_TARGET,
-				error = ?e,
-				"failed to notify collator about seconded collation"
-			);
-		}
+		sender.send_message(
+			CollatorProtocolMessage::NotifyCollationSeconded(received_from.clone(), statement).into()
+		).await;
 	}
 }
 
@@ -401,17 +381,18 @@ async fn get_collation(
 	relay_parent: Hash,
 	para_id: ParaId,
 	collator_id: CollatorId,
-	mut sender: mpsc::Sender<FromJobCommand>,
+	sender: &mut impl SubsystemSender,
 ) -> Result<(CandidateReceipt, PoV), Error> {
 	let (tx, rx) = oneshot::channel();
 	sender
-		.send(AllMessages::from(CollatorProtocolMessage::FetchCollation(
+		.send_message(CollatorProtocolMessage::FetchCollation(
 			relay_parent,
 			collator_id,
 			para_id,
 			tx,
-		)).into())
-		.await?;
+		).into())
+		.await;
+
 	rx.await.map_err(Into::into)
 }
 
@@ -419,45 +400,33 @@ async fn second_candidate(
 	relay_parent: Hash,
 	candidate_receipt: CandidateReceipt,
 	pov: PoV,
-	sender: &mut mpsc::Sender<FromJobCommand>,
+	sender: &mut impl SubsystemSender,
 	metrics: &Metrics,
-) -> Result<(), Error> {
-	match sender
-		.send(AllMessages::from(CandidateBackingMessage::Second(
+) {
+	sender
+		.send_message(CandidateBackingMessage::Second(
 			relay_parent,
 			candidate_receipt,
 			pov,
-		)).into())
-		.await
-	{
-		Err(err) => {
-			tracing::warn!(target: LOG_TARGET, err = ?err, "failed to send a seconding message");
-			metrics.on_second(Err(()));
-			Err(err.into())
-		}
-		Ok(_) => {
-			metrics.on_second(Ok(()));
-			Ok(())
-		}
-	}
+		).into())
+		.await;
+
+	metrics.on_second();
 }
 
 async fn forward_invalidity_note(
 	received_from: &CollatorId,
-	sender: &mut mpsc::Sender<FromJobCommand>,
-) -> Result<(), Error> {
+	sender: &mut impl SubsystemSender,
+) {
 	sender
-		.send(AllMessages::from(CollatorProtocolMessage::ReportCollator(
-			received_from.clone(),
-		)).into())
+		.send_message(CollatorProtocolMessage::ReportCollator(received_from.clone()).into())
 		.await
-		.map_err(Into::into)
 }
 
 #[derive(Clone)]
 struct MetricsInner {
-	seconds: prometheus::CounterVec<prometheus::U64>,
-	invalid_selections: prometheus::CounterVec<prometheus::U64>,
+	seconds: prometheus::Counter<prometheus::U64>,
+	invalid_selections: prometheus::Counter<prometheus::U64>,
 	handle_collation: prometheus::Histogram,
 	handle_invalid: prometheus::Histogram,
 }
@@ -467,17 +436,15 @@ struct MetricsInner {
 pub struct Metrics(Option<MetricsInner>);
 
 impl Metrics {
-	fn on_second(&self, result: Result<(), ()>) {
+	fn on_second(&self) {
 		if let Some(metrics) = &self.0 {
-			let label = if result.is_ok() { "succeeded" } else { "failed" };
-			metrics.seconds.with_label_values(&[label]).inc();
+			metrics.seconds.inc();
 		}
 	}
 
-	fn on_invalid_selection(&self, result: Result<(), ()>) {
+	fn on_invalid_selection(&self) {
 		if let Some(metrics) = &self.0 {
-			let label = if result.is_ok() { "succeeded" } else { "failed" };
-			metrics.invalid_selections.with_label_values(&[label]).inc();
+			metrics.invalid_selections.inc();
 		}
 	}
 
@@ -496,22 +463,20 @@ impl metrics::Metrics for Metrics {
 	fn try_register(registry: &prometheus::Registry) -> Result<Self, prometheus::PrometheusError> {
 		let metrics = MetricsInner {
 			seconds: prometheus::register(
-				prometheus::CounterVec::new(
+				prometheus::Counter::with_opts(
 					prometheus::Opts::new(
 						"candidate_selection_seconds_total",
 						"Number of Candidate Selection subsystem seconding events.",
 					),
-					&["success"],
 				)?,
 				registry,
 			)?,
 			invalid_selections: prometheus::register(
-				prometheus::CounterVec::new(
+				prometheus::Counter::with_opts(
 					prometheus::Opts::new(
 						"candidate_selection_invalid_selections_total",
 						"Number of Candidate Selection subsystem seconding selections which proved to be invalid.",
 					),
-					&["success"],
 				)?,
 				registry,
 			)?,
@@ -538,13 +503,15 @@ impl metrics::Metrics for Metrics {
 	}
 }
 
-delegated_subsystem!(CandidateSelectionJob(SyncCryptoStorePtr, Metrics) <- CandidateSelectionMessage as CandidateSelectionSubsystem);
+/// The candidate selection subsystem.
+pub type CandidateSelectionSubsystem<Spawner> = JobSubsystem<CandidateSelectionJob, Spawner>;
 
 #[cfg(test)]
 mod tests {
 	use super::*;
 	use futures::lock::Mutex;
 	use polkadot_primitives::v1::BlockData;
+	use polkadot_node_subsystem::messages::AllMessages;
 	use sp_core::crypto::Public;
 	use std::sync::Arc;
 
@@ -554,15 +521,14 @@ mod tests {
 		postconditions: Postconditions,
 	) where
 		Preconditions: FnOnce(&mut CandidateSelectionJob),
-		TestBuilder: FnOnce(mpsc::Sender<CandidateSelectionMessage>, mpsc::Receiver<FromJobCommand>) -> Test,
+		TestBuilder: FnOnce(mpsc::Sender<CandidateSelectionMessage>, mpsc::UnboundedReceiver<AllMessages>) -> Test,
 		Test: Future<Output = ()>,
 		Postconditions: FnOnce(CandidateSelectionJob, Result<(), Error>),
 	{
 		let (to_job_tx, to_job_rx) = mpsc::channel(0);
-		let (from_job_tx, from_job_rx) = mpsc::channel(0);
+		let (mut from_job_tx, from_job_rx) = polkadot_node_subsystem_test_helpers::sender_receiver();
 		let mut job = CandidateSelectionJob {
 			assignment: 123.into(),
-			sender: from_job_tx,
 			receiver: to_job_rx,
 			metrics: Default::default(),
 			seconded_candidate: None,
@@ -570,9 +536,13 @@ mod tests {
 
 		preconditions(&mut job);
 		let span = jaeger::Span::Disabled;
-		let (_, job_result) = futures::executor::block_on(future::join(
+		let (_, (job, job_result)) = futures::executor::block_on(future::join(
 			test(to_job_tx, from_job_rx),
-			job.run_loop(&span),
+			async move {
+				let res = job.run_loop(&span, &mut from_job_tx).await;
+				drop(from_job_tx);
+				(job, res)
+			},
 		));
 
 		postconditions(job, job_result);
@@ -609,12 +579,12 @@ mod tests {
 
 				while let Some(msg) = from_job.next().await {
 					match msg {
-						FromJobCommand::SendMessage(AllMessages::CollatorProtocol(CollatorProtocolMessage::FetchCollation(
+						AllMessages::CollatorProtocol(CollatorProtocolMessage::FetchCollation(
 							got_relay_parent,
 							collator_id,
 							got_para_id,
 							return_sender,
-						))) => {
+						)) => {
 							assert_eq!(got_relay_parent, relay_parent);
 							assert_eq!(got_para_id, para_id);
 							assert_eq!(collator_id, collator_id_clone);
@@ -623,11 +593,11 @@ mod tests {
 								.send((candidate_receipt.clone(), pov.clone()))
 								.unwrap();
 						}
-						FromJobCommand::SendMessage(AllMessages::CandidateBacking(CandidateBackingMessage::Second(
+						AllMessages::CandidateBacking(CandidateBackingMessage::Second(
 							got_relay_parent,
 							got_candidate_receipt,
 							got_pov,
-						))) => {
+						)) => {
 							assert_eq!(got_relay_parent, relay_parent);
 							assert_eq!(got_candidate_receipt, candidate_receipt);
 							assert_eq!(got_pov, pov);
@@ -674,11 +644,11 @@ mod tests {
 
 				while let Some(msg) = from_job.next().await {
 					match msg {
-						FromJobCommand::SendMessage(AllMessages::CandidateBacking(CandidateBackingMessage::Second(
+						AllMessages::CandidateBacking(CandidateBackingMessage::Second(
 							_got_relay_parent,
 							_got_candidate_receipt,
 							_got_pov,
-						))) => {
+						)) => {
 							*was_seconded_clone.lock().await = true;
 						}
 						other => panic!("unexpected message from job: {:?}", other),
@@ -713,13 +683,14 @@ mod tests {
 					.send(CandidateSelectionMessage::Invalid(relay_parent, candidate_receipt))
 					.await
 					.unwrap();
+
 				std::mem::drop(to_job);
 
 				while let Some(msg) = from_job.next().await {
 					match msg {
-						FromJobCommand::SendMessage(AllMessages::CollatorProtocol(CollatorProtocolMessage::ReportCollator(
+						AllMessages::CollatorProtocol(CollatorProtocolMessage::ReportCollator(
 							got_collator_id,
-						))) => {
+						)) => {
 							assert_eq!(got_collator_id, collator_id_clone);
 
 							*sent_report_clone.lock().await = true;

polkadot/node/core/provisioner/Cargo.toml

@@ -17,3 +17,4 @@ futures-timer = "3.0.2"
 [dev-dependencies]
 sp-application-crypto = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" }
+polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }

polkadot/node/core/provisioner/src/lib.rs

@@ -25,14 +25,14 @@ use futures::{
 	prelude::*,
 };
 use polkadot_node_subsystem::{
-	errors::{ChainApiError, RuntimeApiError}, PerLeafSpan, jaeger,
+	errors::{ChainApiError, RuntimeApiError}, PerLeafSpan, SubsystemSender, jaeger,
 	messages::{
-		AllMessages, CandidateBackingMessage, ChainApiMessage, ProvisionableData, ProvisionerInherentData,
+		CandidateBackingMessage, ChainApiMessage, ProvisionableData, ProvisionerInherentData,
 		ProvisionerMessage,
 	},
 };
 use polkadot_node_subsystem_util::{
-	self as util, delegated_subsystem, FromJobCommand,
+	self as util, JobSubsystem, JobSender,
 	request_availability_cores, request_persisted_validation_data, JobTrait, metrics::{self, prometheus},
 };
 use polkadot_primitives::v1::{
@@ -80,9 +80,9 @@ impl InherentAfter {
 	}
 }
 
-struct ProvisioningJob {
+/// A per-relay-parent job for the provisioning subsystem.
+pub struct ProvisioningJob {
 	relay_parent: Hash,
-	sender: mpsc::Sender<FromJobCommand>,
 	receiver: mpsc::Receiver<ProvisionerMessage>,
 	backed_candidates: Vec<CandidateReceipt>,
 	signed_bitfields: Vec<SignedAvailabilityBitfield>,
@@ -91,8 +91,10 @@ struct ProvisioningJob {
 	awaiting_inherent: Vec<oneshot::Sender<ProvisionerInherentData>>
 }
 
+/// Errors in the provisioner.
 #[derive(Debug, Error)]
-enum Error {
+#[allow(missing_docs)]
+pub enum Error {
 	#[error(transparent)]
 	Util(#[from] util::Error),
 
@@ -139,38 +141,35 @@ impl JobTrait for ProvisioningJob {
 	//
 	// this function is in charge of creating and executing the job's main loop
 	#[tracing::instrument(skip(span, _run_args, metrics, receiver, sender), fields(subsystem = LOG_TARGET))]
-	fn run(
+	fn run<S: SubsystemSender>(
 		relay_parent: Hash,
 		span: Arc<jaeger::Span>,
 		_run_args: Self::RunArgs,
 		metrics: Self::Metrics,
 		receiver: mpsc::Receiver<ProvisionerMessage>,
-		sender: mpsc::Sender<FromJobCommand>,
+		mut sender: JobSender<S>,
 	) -> Pin<Box<dyn Future<Output = Result<(), Self::Error>> + Send>> {
 		async move {
 			let job = ProvisioningJob::new(
 				relay_parent,
 				metrics,
-				sender,
 				receiver,
 			);
 
-			job.run_loop(PerLeafSpan::new(span, "provisioner")).await
+			job.run_loop(sender.subsystem_sender(), PerLeafSpan::new(span, "provisioner")).await
 		}
 		.boxed()
 	}
 }
 
 impl ProvisioningJob {
-	pub fn new(
+	fn new(
 		relay_parent: Hash,
 		metrics: Metrics,
-		sender: mpsc::Sender<FromJobCommand>,
 		receiver: mpsc::Receiver<ProvisionerMessage>,
 	) -> Self {
 		Self {
 			relay_parent,
-			sender,
 			receiver,
 			backed_candidates: Vec::new(),
 			signed_bitfields: Vec::new(),
@@ -180,7 +179,11 @@ impl ProvisioningJob {
 		}
 	}
 
-	async fn run_loop(mut self, span: PerLeafSpan) -> Result<(), Error> {
+	async fn run_loop(
+		mut self,
+		sender: &mut impl SubsystemSender,
+		span: PerLeafSpan,
+	) -> Result<(), Error> {
 		use ProvisionerMessage::{
 			ProvisionableData, RequestInherentData,
 		};
@@ -192,7 +195,7 @@ impl ProvisioningJob {
 						let _timer = self.metrics.time_request_inherent_data();
 
 						if self.inherent_after.is_ready() {
-							self.send_inherent_data(vec![return_sender]).await;
+							self.send_inherent_data(sender, vec![return_sender]).await;
 						} else {
 							self.awaiting_inherent.push(return_sender);
 						}
@@ -209,7 +212,7 @@ impl ProvisioningJob {
 					let _span = span.child("send-inherent-data");
 					let return_senders = std::mem::take(&mut self.awaiting_inherent);
 					if !return_senders.is_empty() {
-						self.send_inherent_data(return_senders).await;
+						self.send_inherent_data(sender, return_senders).await;
 					}
 				}
 			}
@@ -220,6 +223,7 @@ impl ProvisioningJob {
 
 	async fn send_inherent_data(
 		&mut self,
+		sender: &mut impl SubsystemSender,
 		return_senders: Vec<oneshot::Sender<ProvisionerInherentData>>,
 	) {
 		if let Err(err) = send_inherent_data(
@@ -227,7 +231,7 @@ impl ProvisioningJob {
 			&self.signed_bitfields,
 			&self.backed_candidates,
 			return_senders,
-			&mut self.sender,
+			sender,
 		)
 		.await
 		{
@@ -279,10 +283,10 @@ async fn send_inherent_data(
 	bitfields: &[SignedAvailabilityBitfield],
 	candidates: &[CandidateReceipt],
 	return_senders: Vec<oneshot::Sender<ProvisionerInherentData>>,
-	from_job: &mut mpsc::Sender<FromJobCommand>,
+	from_job: &mut impl SubsystemSender,
 ) -> Result<(), Error> {
 	let availability_cores = request_availability_cores(relay_parent, from_job)
-		.await?
+		.await
 		.await.map_err(|err| Error::CanceledAvailabilityCores(err))??;
 
 	let bitfields = select_availability_bitfields(&availability_cores, bitfields);
@@ -351,7 +355,7 @@ async fn select_candidates(
 	bitfields: &[SignedAvailabilityBitfield],
 	candidates: &[CandidateReceipt],
 	relay_parent: Hash,
-	sender: &mut mpsc::Sender<FromJobCommand>,
+	sender: &mut impl SubsystemSender,
 ) -> Result<Vec<BackedCandidate>, Error> {
 	let block_number = get_block_number_under_construction(relay_parent, sender).await?;
 
@@ -388,7 +392,7 @@ async fn select_candidates(
 			assumption,
 			sender,
 		)
-		.await?
+		.await
 		.await.map_err(|err| Error::CanceledPersistedValidationData(err))??
 		{
 			Some(v) => v,
@@ -418,11 +422,11 @@ async fn select_candidates(
 
 	// now get the backed candidates corresponding to these candidate receipts
 	let (tx, rx) = oneshot::channel();
-	sender.send(AllMessages::CandidateBacking(CandidateBackingMessage::GetBackedCandidates(
+	sender.send_message(CandidateBackingMessage::GetBackedCandidates(
 		relay_parent,
 		selected_candidates.clone(),
 		tx,
-	)).into()).await.map_err(|err| Error::GetBackedCandidatesSend(err))?;
+	).into()).await;
 	let mut candidates = rx.await.map_err(|err| Error::CanceledBackedCandidates(err))?;
 
 	// `selected_candidates` is generated in ascending order by core index, and `GetBackedCandidates`
@@ -470,16 +474,16 @@ async fn select_candidates(
 #[tracing::instrument(level = "trace", skip(sender), fields(subsystem = LOG_TARGET))]
 async fn get_block_number_under_construction(
 	relay_parent: Hash,
-	sender: &mut mpsc::Sender<FromJobCommand>,
+	sender: &mut impl SubsystemSender,
 ) -> Result<BlockNumber, Error> {
 	let (tx, rx) = oneshot::channel();
 	sender
-		.send(AllMessages::from(ChainApiMessage::BlockNumber(
+		.send_message(ChainApiMessage::BlockNumber(
 			relay_parent,
 			tx,
-		)).into())
-		.await
-		.map_err(|e| Error::ChainApiMessageSend(e))?;
+		).into())
+		.await;
+
 	match rx.await.map_err(|err| Error::CanceledBlockNumber(err))? {
 		Ok(Some(n)) => Ok(n + 1),
 		Ok(None) => Ok(0),
@@ -596,7 +600,8 @@ impl metrics::Metrics for Metrics {
 }
 
 
-delegated_subsystem!(ProvisioningJob((), Metrics) <- ProvisionerMessage as ProvisioningSubsystem);
+/// The provisioning subsystem.
+pub type ProvisioningSubsystem<Spawner> = JobSubsystem<ProvisioningJob, Spawner>;
 
 #[cfg(test)]
 mod tests;

polkadot/node/core/provisioner/src/tests.rs

@@ -191,7 +191,6 @@ mod select_availability_bitfields {
 }
 
 mod select_candidates {
-	use futures_timer::Delay;
 	use super::super::*;
 	use super::{build_occupied_core, occupied_core, scheduled_core, default_bitvec};
 	use polkadot_node_subsystem::messages::{
@@ -201,6 +200,7 @@ mod select_candidates {
 	use polkadot_primitives::v1::{
 		BlockNumber, CandidateDescriptor, PersistedValidationData, CommittedCandidateReceipt, CandidateCommitments,
 	};
+	use polkadot_node_subsystem_test_helpers::TestSubsystemSender;
 
 	const BLOCK_UNDER_PRODUCTION: BlockNumber = 128;
 
@@ -208,12 +208,12 @@ mod select_candidates {
 		overseer_factory: OverseerFactory,
 		test_factory: TestFactory,
 	) where
-		OverseerFactory: FnOnce(mpsc::Receiver<FromJobCommand>) -> Overseer,
+		OverseerFactory: FnOnce(mpsc::UnboundedReceiver<AllMessages>) -> Overseer,
 		Overseer: Future<Output = ()>,
-		TestFactory: FnOnce(mpsc::Sender<FromJobCommand>) -> Test,
+		TestFactory: FnOnce(TestSubsystemSender) -> Test,
 		Test: Future<Output = ()>,
 	{
-		let (tx, rx) = mpsc::channel(64);
+		let (tx, rx) = polkadot_node_subsystem_test_helpers::sender_receiver();
 		let overseer = overseer_factory(rx);
 		let test = test_factory(tx);
 
@@ -298,24 +298,27 @@ mod select_candidates {
 		]
 	}
 
-	async fn mock_overseer(mut receiver: mpsc::Receiver<FromJobCommand>, expected: Vec<BackedCandidate>) {
+	async fn mock_overseer(
+		mut receiver: mpsc::UnboundedReceiver<AllMessages>,
+		expected: Vec<BackedCandidate>,
+	) {
 		use ChainApiMessage::BlockNumber;
 		use RuntimeApiMessage::Request;
 
 		while let Some(from_job) = receiver.next().await {
 			match from_job {
-				FromJobCommand::SendMessage(AllMessages::ChainApi(BlockNumber(_relay_parent, tx))) => {
+				AllMessages::ChainApi(BlockNumber(_relay_parent, tx)) => {
 					tx.send(Ok(Some(BLOCK_UNDER_PRODUCTION - 1))).unwrap()
 				}
-				FromJobCommand::SendMessage(AllMessages::RuntimeApi(Request(
+				AllMessages::RuntimeApi(Request(
 					_parent_hash,
 					PersistedValidationDataReq(_para_id, _assumption, tx),
-				))) => tx.send(Ok(Some(Default::default()))).unwrap(),
-				FromJobCommand::SendMessage(AllMessages::RuntimeApi(Request(_parent_hash, AvailabilityCores(tx)))) => {
+				)) => tx.send(Ok(Some(Default::default()))).unwrap(),
+				AllMessages::RuntimeApi(Request(_parent_hash, AvailabilityCores(tx))) => {
 					tx.send(Ok(mock_availability_cores())).unwrap()
 				}
-				FromJobCommand::SendMessage(
-					AllMessages::CandidateBacking(CandidateBackingMessage::GetBackedCandidates(_, _, sender))
+				AllMessages::CandidateBacking(
+					CandidateBackingMessage::GetBackedCandidates(_, _, sender)
 				) => {
 					let _ = sender.send(expected.clone());
 				}
@@ -324,29 +327,9 @@ mod select_candidates {
 		}
 	}
 
-	#[test]
-	fn handles_overseer_failure() {
-		let overseer = |rx: mpsc::Receiver<FromJobCommand>| async move {
-			// drop the receiver so it closes and the sender can't send, then just sleep long enough that
-			// this is almost certainly not the first of the two futures to complete
-			std::mem::drop(rx);
-			Delay::new(std::time::Duration::from_secs(1)).await;
-		};
-
-		let test = |mut tx: mpsc::Sender<FromJobCommand>| async move {
-			// wait so that the overseer can drop the rx before we attempt to send
-			Delay::new(std::time::Duration::from_millis(50)).await;
-			let result = select_candidates(&[], &[], &[], Default::default(), &mut tx).await;
-			println!("{:?}", result);
-			assert!(std::matches!(result, Err(Error::ChainApiMessageSend(_))));
-		};
-
-		test_harness(overseer, test);
-	}
-
 	#[test]
 	fn can_succeed() {
-		test_harness(|r| mock_overseer(r, Vec::new()), |mut tx: mpsc::Sender<FromJobCommand>| async move {
+		test_harness(|r| mock_overseer(r, Vec::new()), |mut tx: TestSubsystemSender| async move {
 			select_candidates(&[], &[], &[], Default::default(), &mut tx).await.unwrap();
 		})
 	}
@@ -411,7 +394,7 @@ mod select_candidates {
 			})
 			.collect();
 
-		test_harness(|r| mock_overseer(r, expected_backed), |mut tx: mpsc::Sender<FromJobCommand>| async move {
+		test_harness(|r| mock_overseer(r, expected_backed), |mut tx: TestSubsystemSender| async move {
 			let result =
 				select_candidates(&mock_cores, &[], &candidates, Default::default(), &mut tx)
 					.await.unwrap();
@@ -470,7 +453,7 @@ mod select_candidates {
 			})
 			.collect();
 
-		test_harness(|r| mock_overseer(r, expected_backed), |mut tx: mpsc::Sender<FromJobCommand>| async move {
+		test_harness(|r| mock_overseer(r, expected_backed), |mut tx: TestSubsystemSender| async move {
 			let result =
 				select_candidates(&mock_cores, &[], &candidates, Default::default(), &mut tx)
 					.await.unwrap();