Overseer: subsystems communicate directly (#2227)

* overseer: pass messages directly between subsystems

* test that message is held on to

* Update node/overseer/src/lib.rs

Co-authored-by: Peter Goodspeed-Niklaus <coriolinus@users.noreply.github.com>

* give every subsystem an unbounded sender too

* remove metered_channel::name

1. we don't provide good names
2. these names are never used anywhere

* unused mut

* remove unnecessary &mut

* subsystem unbounded_send

* remove unused MaybeTimer

We have channel size metrics that serve the same purpose better now and the implementation of message timing was pretty ugly.

* remove comment

* split up senders and receivers

* update metrics

* fix tests

* fix test subsystem context

* fix flaky test

* fix docs

* doc

* use select_biased to favor signals

* Update node/subsystem/src/lib.rs

Co-authored-by: Andronik Ordian <write@reusable.software>

Co-authored-by: Peter Goodspeed-Niklaus <coriolinus@users.noreply.github.com>
Co-authored-by: Andronik Ordian <write@reusable.software>

polkadot/node/subsystem/src/lib.rs

@@ -210,6 +210,27 @@ pub struct SpawnedSubsystem {
 /// [`SubsystemError`]: struct.SubsystemError.html
 pub type SubsystemResult<T> = Result<T, SubsystemError>;
 
+/// A sender used by subsystems to communicate with other subsystems.
+///
+/// Each clone of this type may add more capacity to the bounded buffer, so clones should
+/// be used sparingly.
+#[async_trait]
+pub trait SubsystemSender: Send + Clone + 'static {
+	/// Send a direct message to some other `Subsystem`, routed based on message type.
+	async fn send_message(&mut self, msg: AllMessages);
+
+	/// Send multiple direct messages to other `Subsystem`s, routed based on message type.
+	async fn send_messages<T>(&mut self, msgs: T)
+		where T: IntoIterator<Item = AllMessages> + Send, T::IntoIter: Send;
+
+	/// Send a message onto the unbounded queue of some other `Subsystem`, routed based on message
+	/// type.
+	///
+	/// This function should be used only when there is some other bounding factor on the messages
+	/// sent with it. Otherwise, it risks a memory leak.
+	fn send_unbounded_message(&mut self, msg: AllMessages);
+}
+
 /// A context type that is given to the [`Subsystem`] upon spawning.
 /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s
 /// or spawn jobs.
@@ -217,11 +238,14 @@ pub type SubsystemResult<T> = Result<T, SubsystemError>;
 /// [`Overseer`]: struct.Overseer.html
 /// [`SubsystemJob`]: trait.SubsystemJob.html
 #[async_trait]
-pub trait SubsystemContext: Send + 'static {
+pub trait SubsystemContext: Send + Sized + 'static {
 	/// The message type of this context. Subsystems launched with this context will expect
 	/// to receive messages of this type.
 	type Message: Send;
 
+	/// The message sender type of this context. Clones of the sender should be used sparingly.
+	type Sender: SubsystemSender;
+
 	/// Try to asynchronously receive a message.
 	///
 	/// This has to be used with caution, if you loop over this without
@@ -241,12 +265,34 @@ pub trait SubsystemContext: Send + 'static {
 		s: Pin<Box<dyn Future<Output = ()> + Send>>,
 	) -> SubsystemResult<()>;
 
+	/// Get a mutable reference to the sender.
+	fn sender(&mut self) -> &mut Self::Sender;
+
 	/// Send a direct message to some other `Subsystem`, routed based on message type.
-	async fn send_message(&mut self, msg: AllMessages);
+	async fn send_message(&mut self, msg: AllMessages) {
+		self.sender().send_message(msg).await
+	}
 
 	/// Send multiple direct messages to other `Subsystem`s, routed based on message type.
 	async fn send_messages<T>(&mut self, msgs: T)
-		where T: IntoIterator<Item = AllMessages> + Send, T::IntoIter: Send;
+		where T: IntoIterator<Item = AllMessages> + Send, T::IntoIter: Send
+	{
+		self.sender().send_messages(msgs).await
+	}
+
+
+	/// Send a message onto the unbounded queue of some other `Subsystem`, routed based on message
+	/// type.
+	///
+	/// This function should be used only when there is some other bounding factor on the messages
+	/// sent with it. Otherwise, it risks a memory leak.
+	///
+	/// Generally, for this method to be used, these conditions should be met:
+	/// * There is a communication cycle between subsystems
+	/// * One of the parts of the cycle has a clear bound on the number of messages produced.
+	fn send_unbounded_message(&mut self, msg: AllMessages) {
+		self.sender().send_unbounded_message(msg)
+	}
 }
 
 /// A trait that describes the [`Subsystem`]s that can run on the [`Overseer`].