refactor+feat: allow subsystems to send only declared messages, generate graphviz (#5314)

Closes #3774
Closes #3826

polkadot/node/overseer/overseer-gen/Cargo.toml

@@ -4,6 +4,7 @@ version = "0.9.19"
 authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2021"
 description = "Generate an overseer including builder pattern and message wrapper from a single struct."
+autoexamples = false
 
 [dependencies]
 gum = { package = "tracing-gum", path = "../../gum" }
@@ -22,6 +23,16 @@ pin-project = "1.0"
 trybuild = "1.0.61"
 rustversion = "1.0.6"
 
+
+
+[[example]]
+name = "duo"
+crate-type = ["bin"]
+
+[[example]]
+name = "solo"
+crate-type = ["bin"]
+
 [features]
 default = []
 expand = ["polkadot-overseer-gen-proc-macro/expand"]

polkadot/node/overseer/overseer-gen/README.md

@@ -11,17 +11,20 @@ declarative.
 ```rust
     #[overlord(signal=SigSigSig, event=Event, gen=AllMessages, error=OverseerError)]
     pub struct Overseer {
-        #[subsystem(MsgA)]
+        #[subsystem(MsgA, sends: [MsgB])]
         sub_a: AwesomeSubSysA,
 
-        #[subsystem(MsgB)]
+        #[subsystem(MsgB, sends: [MsgA])]
         sub_b: AwesomeSubSysB,
     }
 ```
 
 * Each subsystem is annotated with `#[subsystem(_)]` where `MsgA` respectively `MsgB` are the messages
 being consumed by that particular subsystem. Each of those subsystems is required to implement the subsystem
-trait.
+trait with the correct trait bounds. Commonly this is achieved
+by using `#[subsystem]` and `#[contextbounds]` macro.
+  * `#[contextbounds(Foo, error=Yikes, prefix=wherethetraitsat)]` can applied to `impl`-blocks and `fn`-blocks. It will add additional trait bounds for the generic `Context` with `Context: FooContextTrait` for `<Context as FooContextTrait>::Sender: FooSenderTrait` besides a few more. Note that `Foo` here references the name of the subsystem as declared in `#[overlord(..)]` macro.
+  * `#[subsystem(Foo, error=Yikes, prefix=wherethetraitsat)]` is a extension to the above, implementing `trait Subsystem<Context, Yikes>`.
 * `error=` tells the overseer to use the user provided
 error type, if not provided a builtin one is used. Note that this is the one error type used throughout all calls, so make sure it does impl `From<E>` for all other error types `E` that are relevant to your application.
 * `event=` declares an external event type, that injects certain events
@@ -63,10 +66,10 @@ is not ready to be included in the Overseer:
 ```rust
     #[overlord(signal=SigSigSig, event=Event, gen=AllMessages, error=OverseerError)]
     pub struct Overseer {
-        #[subsystem(MsgA)]
+        #[subsystem(MsgA, sends: MsgB)]
         sub_a: AwesomeSubSysA,
 
-        #[subsystem(MsgB), wip]
+        #[subsystem(MsgB, sends: MsgA), wip]
         sub_b: AwesomeSubSysB, // This subsystem will not be required nor allowed to be set
     }
 ```

polkadot/node/overseer/overseer-gen/adr/01-adr.md

@@ -0,0 +1,21 @@
+# Limit outgoing messages
+
+## Status
+
+Accepted + implemented.
+
+## Context
+
+Previously, there was no way to limit and hence reason about a subset of subsystems, and if they form a cycle. Limiting the outgoing message types is a first step to create respective graphs and use classic graph algorithms to detect those and leave it to the user to resolve these.
+
+## Decision
+
+Annotate the `#[overlord]` inner `#[subsystem(..)]` annotation
+with an aditional set of outgoing messages and enforce this via more fine grained trait bounds on the `Sender` and `<Context>::Sender` bounds.
+
+## Consequences
+
+* A graph will be spawn for every compilation under the `OUT_DIR` of the crate where `#[overlord]` is specified.
+* Each subsystem has a consuming message which is often referred to as generic `M` (no change on that, is as before), but now we have trait `AssociateOutgoing { type OutgoingMessages = ..; }` which defines an outgoing helper `enum` that is generated with an ident constructed as `${Subsystem}OutgoingMessages` where `${Subsystem}` is the subsystem identifier as used in the overseer declaration. `${Subsystem}OutgoingMessages` is used throughout everywhere to constrain the outgoing messages (commonly referred to as `OutgoingMessage` generic bounded by `${Subsystem}OutgoingMessages: From<OutgoingMessage>` or `::OutgoingMessages: From`. It's what allows the construction of the graph and compile time verification.
+* `${Subsystem}SenderTrait` and `${Subsystem}ContextTrait` are accumulation traits or wrapper traits, that combine over all annotated M or `OutgoingMessages` from the overseer declaration or their respective outgoing types. It is usage convenience and assures consistency within a subsystem while also maintaining a single source of truth for which messages can be sent by a particular subsystem. Note that this is sidestepped for the test subsystem, which may consume `gen=AllMessages`, the global message wrapper type.
+* `Job`-based subsystems, being on their way out, are patched, but they now are generic over the `Sender` type, leaking that type.

polkadot/node/overseer/overseer-gen/examples/dummy.rs

@@ -1,143 +0,0 @@
-//! A dummy to be used with cargo expand
-
-use polkadot_node_network_protocol::WrongVariant;
-use polkadot_overseer_gen::*;
-use std::collections::HashMap;
-
-/// Concrete subsystem implementation for `MsgStrukt` msg type.
-#[derive(Default)]
-pub struct AwesomeSubSys;
-
-impl ::polkadot_overseer_gen::Subsystem<XxxSubsystemContext<MsgStrukt>, Yikes> for AwesomeSubSys {
-	fn start(self, _ctx: XxxSubsystemContext<MsgStrukt>) -> SpawnedSubsystem<Yikes> {
-		unimplemented!("starting yay!")
-	}
-}
-
-#[derive(Default)]
-pub struct GoblinTower;
-
-impl ::polkadot_overseer_gen::Subsystem<XxxSubsystemContext<Plinko>, Yikes> for GoblinTower {
-	fn start(self, _ctx: XxxSubsystemContext<Plinko>) -> SpawnedSubsystem<Yikes> {
-		unimplemented!("welcum")
-	}
-}
-
-/// A signal sent by the overseer.
-#[derive(Debug, Clone)]
-pub struct SigSigSig;
-
-/// The external event.
-#[derive(Debug, Clone)]
-pub struct EvX;
-
-impl EvX {
-	pub fn focus<'a, T>(&'a self) -> Result<EvX, ()> {
-		unimplemented!("dispatch")
-	}
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct Yikes;
-
-impl std::fmt::Display for Yikes {
-	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		writeln!(f, "yikes!")
-	}
-}
-
-impl std::error::Error for Yikes {}
-
-impl From<polkadot_overseer_gen::OverseerError> for Yikes {
-	fn from(_: polkadot_overseer_gen::OverseerError) -> Yikes {
-		Yikes
-	}
-}
-
-impl From<polkadot_overseer_gen::mpsc::SendError> for Yikes {
-	fn from(_: polkadot_overseer_gen::mpsc::SendError) -> Yikes {
-		Yikes
-	}
-}
-
-#[derive(Debug, Clone)]
-pub struct MsgStrukt(u8);
-
-#[derive(Debug, Clone, Copy)]
-pub struct Plinko;
-
-impl From<NetworkMsg> for MsgStrukt {
-	fn from(_event: NetworkMsg) -> Self {
-		MsgStrukt(1u8)
-	}
-}
-
-#[derive(Debug, Clone, Copy)]
-pub enum NetworkMsg {
-	A,
-	B,
-	C,
-}
-
-impl NetworkMsg {
-	fn focus(&self) -> Result<Self, WrongVariant> {
-		Ok(match self {
-			Self::B => return Err(WrongVariant),
-			Self::A | Self::C => self.clone(),
-		})
-	}
-}
-
-#[overlord(signal=SigSigSig, event=EvX, error=Yikes, network=NetworkMsg, gen=AllMessages)]
-struct Xxx<T> {
-	#[subsystem(MsgStrukt)]
-	sub0: AwesomeSubSys,
-
-	#[subsystem(no_dispatch, blocking, Plinko)]
-	plinkos: GoblinTower,
-
-	i_like_pi: f64,
-	i_like_generic: T,
-	i_like_hash: HashMap<f64, f64>,
-}
-
-#[derive(Debug, Clone)]
-struct DummySpawner;
-
-impl SpawnNamed for DummySpawner {
-	fn spawn_blocking(
-		&self,
-		task_name: &'static str,
-		subsystem_name: Option<&'static str>,
-		_future: futures::future::BoxFuture<'static, ()>,
-	) {
-		unimplemented!("spawn blocking {} {}", task_name, subsystem_name.unwrap_or("default"))
-	}
-
-	fn spawn(
-		&self,
-		task_name: &'static str,
-		subsystem_name: Option<&'static str>,
-		_future: futures::future::BoxFuture<'static, ()>,
-	) {
-		unimplemented!("spawn {} {}", task_name, subsystem_name.unwrap_or("default"))
-	}
-}
-
-#[derive(Debug, Clone)]
-struct DummyCtx;
-
-fn main() {
-	let (overseer, _handle): (Xxx<_, f64>, _) = Xxx::builder()
-		.sub0(AwesomeSubSys::default())
-		.plinkos(GoblinTower::default())
-		.i_like_pi(::std::f64::consts::PI)
-		.i_like_generic(42.0)
-		.i_like_hash(HashMap::new())
-		.spawner(DummySpawner)
-		.build()
-		.unwrap();
-	assert_eq!(overseer.i_like_pi.floor() as i8, 3);
-	assert_eq!(overseer.i_like_generic.floor() as i8, 42);
-	assert_eq!(overseer.i_like_hash.len() as i8, 0);
-}

polkadot/node/overseer/overseer-gen/examples/duo.rs

@@ -0,0 +1,89 @@
+#![allow(dead_code)] // overseer events are not used
+
+//! A dummy to be used with cargo expand
+
+use polkadot_overseer_gen::{self as overseer, SpawnNamed, *};
+use std::collections::HashMap;
+mod misc;
+
+pub use self::misc::*;
+
+/// Concrete subsystem implementation for `MsgStrukt` msg type.
+#[derive(Default)]
+pub struct AwesomeSubSys;
+
+#[overseer::subsystem(Awesome, error=Yikes)]
+impl<Context> AwesomeSubSys {
+	fn start(self, mut ctx: Context) -> SpawnedSubsystem<Yikes> {
+		let mut sender = ctx.sender().clone();
+		ctx.spawn(
+			"AwesomeSubsys",
+			Box::pin(async move {
+				sender.send_message(Plinko).await;
+			}),
+		)
+		.unwrap();
+		unimplemented!("starting yay!")
+	}
+}
+
+#[derive(Default)]
+pub struct Fortified;
+
+#[overseer::subsystem(GoblinTower, error=Yikes)]
+impl<Context> Fortified {
+	fn start(self, mut ctx: Context) -> SpawnedSubsystem<Yikes> {
+		let mut sender = ctx.sender().clone();
+		ctx.spawn(
+			"GoblinTower",
+			Box::pin(async move {
+				sender.send_message(MsgStrukt(8u8)).await;
+			}),
+		)
+		.unwrap();
+		unimplemented!("welcum")
+	}
+}
+
+#[overlord(signal=SigSigSig, event=EvX, error=Yikes, gen=AllMessages)]
+struct Duo<T> {
+	#[subsystem(consumes: MsgStrukt, sends: [Plinko])]
+	sub0: Awesome,
+
+	#[subsystem(blocking, consumes: Plinko, sends: [MsgStrukt])]
+	plinkos: GoblinTower,
+
+	i_like_pi: f64,
+	i_like_generic: T,
+	i_like_hash: HashMap<f64, f64>,
+}
+
+fn main() {
+	use futures::{executor, pin_mut};
+
+	executor::block_on(async move {
+		let (overseer, _handle): (Duo<_, f64>, _) = Duo::builder()
+			.sub0(AwesomeSubSys::default())
+			.plinkos(Fortified::default())
+			.i_like_pi(::std::f64::consts::PI)
+			.i_like_generic(42.0)
+			.i_like_hash(HashMap::new())
+			.spawner(DummySpawner)
+			.build()
+			.unwrap();
+
+		assert_eq!(overseer.i_like_pi.floor() as i8, 3);
+		assert_eq!(overseer.i_like_generic.floor() as i8, 42);
+		assert_eq!(overseer.i_like_hash.len() as i8, 0);
+
+		let overseer_fut = overseer
+			.running_subsystems
+			.into_future()
+			.timeout(std::time::Duration::from_millis(300))
+			.fuse();
+
+		pin_mut!(overseer_fut);
+
+		overseer_fut.await
+	});
+}

polkadot/node/overseer/overseer-gen/examples/misc.rs

@@ -0,0 +1,69 @@
+use polkadot_overseer_gen::{SpawnNamed, *};
+
+#[derive(Debug, Clone, Copy)]
+pub enum SigSigSig {
+	Conclude,
+	Foo,
+}
+
+#[derive(Debug, Clone)]
+pub struct DummySpawner;
+
+impl SpawnNamed for DummySpawner {
+	fn spawn_blocking(
+		&self,
+		task_name: &'static str,
+		subsystem_name: Option<&'static str>,
+		_future: futures::future::BoxFuture<'static, ()>,
+	) {
+		unimplemented!("spawn blocking {} {}", task_name, subsystem_name.unwrap_or("default"))
+	}
+
+	fn spawn(
+		&self,
+		task_name: &'static str,
+		subsystem_name: Option<&'static str>,
+		_future: futures::future::BoxFuture<'static, ()>,
+	) {
+		unimplemented!("spawn {} {}", task_name, subsystem_name.unwrap_or("default"))
+	}
+}
+
+/// The external event.
+#[derive(Debug, Clone)]
+pub struct EvX;
+
+impl EvX {
+	pub fn focus<'a, T>(&'a self) -> Result<EvX, ()> {
+		unimplemented!("focus")
+	}
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct Yikes;
+
+impl std::fmt::Display for Yikes {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		writeln!(f, "yikes!")
+	}
+}
+
+impl std::error::Error for Yikes {}
+
+impl From<polkadot_overseer_gen::OverseerError> for Yikes {
+	fn from(_: polkadot_overseer_gen::OverseerError) -> Yikes {
+		Yikes
+	}
+}
+
+impl From<polkadot_overseer_gen::mpsc::SendError> for Yikes {
+	fn from(_: polkadot_overseer_gen::mpsc::SendError) -> Yikes {
+		Yikes
+	}
+}
+
+#[derive(Debug, Clone)]
+pub struct MsgStrukt(pub u8);
+
+#[derive(Debug, Clone, Copy)]
+pub struct Plinko;

polkadot/node/overseer/overseer-gen/examples/solo.rs

@@ -0,0 +1,54 @@
+#![allow(dead_code)] // overseer events are not used
+
+//! A minimal demo to be used with cargo expand.
+
+use polkadot_overseer_gen::{self as overseer, SpawnNamed, *};
+mod misc;
+
+pub use self::misc::*;
+
+#[overlord(signal=SigSigSig, event=EvX, error=Yikes, gen=AllMessages)]
+struct Solo<T> {
+	#[subsystem(consumes: Plinko, sends: [MsgStrukt])]
+	goblin_tower: GoblinTower,
+}
+
+#[derive(Default)]
+pub struct Fortified;
+
+#[overseer::subsystem(GoblinTower, error=Yikes)]
+impl<Context> Fortified {
+	fn start(self, mut ctx: Context) -> SpawnedSubsystem<Yikes> {
+		let mut sender = ctx.sender().clone();
+		ctx.spawn(
+			"GoblinTower",
+			Box::pin(async move {
+				sender.send_message(MsgStrukt(8u8)).await;
+			}),
+		)
+		.unwrap();
+		unimplemented!("welcum")
+	}
+}
+
+fn main() {
+	use futures::{executor, pin_mut};
+
+	executor::block_on(async move {
+		let (overseer, _handle): (Solo<_>, _) = Solo::builder()
+			.goblin_tower(Fortified::default())
+			.spawner(DummySpawner)
+			.build()
+			.unwrap();
+
+		let overseer_fut = overseer
+			.running_subsystems
+			.into_future()
+			.timeout(std::time::Duration::from_millis(300))
+			.fuse();
+
+		pin_mut!(overseer_fut);
+
+		overseer_fut.await
+	});
+}

polkadot/node/overseer/overseer-gen/proc-macro/Cargo.toml

@@ -17,12 +17,19 @@ quote = "1.0.18"
 proc-macro2 = "1.0.37"
 proc-macro-crate = "1.1.3"
 expander = { version = "0.0.6", default-features = false }
+petgraph = "0.6.0"
 
 [dev-dependencies]
 assert_matches = "1.5.0"
+polkadot-overseer-gen = { path = "../" }
+thiserror = "1"
+gum = { package = "tracing-gum", path = "../../../gum" }
 
 [features]
-default = []
+default = ["graph", "expand"]
 # write the expanded version to a `overlord-expansion.[a-f0-9]{10}.rs`
 # in the `OUT_DIR` as defined by `cargo` for the `expander` crate.
 expand = []
+# Create directional message consuming / outgoing graph.
+# Generates: `${OUT_DIR}/${overseer|lowercase}-subsystem-messaging.dot`
+graph = []

polkadot/node/overseer/overseer-gen/proc-macro/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+	// populate OUT_DIR
+}

polkadot/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs

@@ -103,7 +103,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream {
 		info.subsystems().iter().filter(|ssf| !ssf.wip).enumerate().map(|(idx, ssf)| {
 			let field_name = &ssf.name;
 			let field_type = &ssf.generic;
-			let subsystem_consumes = &ssf.consumes;
+			let subsystem_consumes = &ssf.message_to_consume;
 			// Remove state generic for the item to be replaced. It sufficient to know `field_type` for
 			// that since we always move from `Init<#field_type>` to `Init<NEW>`.
 			let impl_subsystem_state_generics = recollect_without_idx(&subsystem_passthrough_state_generics[..], idx);
@@ -130,19 +130,28 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream {
 			// see the loop below.
 			let to_keep_subsystem_name = recollect_without_idx(&subsystem_name[..], idx);
 
+			let subsystem_sender_trait = format_ident!("{}SenderTrait", field_type);
+			let _subsystem_ctx_trait = format_ident!("{}ContextTrait", field_type);
+
+			let builder_where_clause = quote!{
+				#field_type : #support_crate::Subsystem< #subsystem_ctx_name< #subsystem_consumes >, #error_ty>,
+				< #subsystem_ctx_name < #subsystem_consumes > as #support_crate :: SubsystemContext>::Sender:
+					#subsystem_sender_trait,
+			};
+
 			// Create the field init `fn`
 			quote! {
 				impl <InitStateSpawner, #field_type, #( #impl_subsystem_state_generics, )* #( #baggage_passthrough_state_generics, )*>
 				#builder <InitStateSpawner, #( #current_state_generics, )* #( #baggage_passthrough_state_generics, )*>
 				where
-					#field_type : Subsystem<#subsystem_ctx_name<#subsystem_consumes>, #error_ty>,
+					#builder_where_clause
 				{
 					/// Specify the subsystem in the builder directly
 					pub fn #field_name (self, var: #field_type ) ->
 						#builder <InitStateSpawner, #( #post_setter_state_generics, )* #( #baggage_passthrough_state_generics, )*>
 					{
 						#builder {
-							#field_name: Init::<#field_type>::Value(var),
+							#field_name: Init::< #field_type >::Value(var),
 							#(
 								#to_keep_subsystem_name: self. #to_keep_subsystem_name,
 							)*
@@ -158,7 +167,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream {
 					/// Specify the the initialization function for a subsystem
 					pub fn #field_name_with<'a, F>(self, subsystem_init_fn: F ) ->
 						#builder <InitStateSpawner, #( #post_setter_state_generics, )* #( #baggage_passthrough_state_generics, )*>
-						where
+					where
 						F: 'static + FnOnce(#handle) ->
 							::std::result::Result<#field_type, #error_ty>,
 					{
@@ -185,7 +194,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream {
 				impl <InitStateSpawner, #field_type, #( #impl_subsystem_state_generics, )* #( #baggage_passthrough_state_generics, )*>
 				#builder <InitStateSpawner, #( #post_setter_state_generics, )* #( #baggage_passthrough_state_generics, )*>
 				where
-					#field_type : Subsystem<#subsystem_ctx_name<#subsystem_consumes>, #error_ty>,
+					#builder_where_clause
 				{
 					/// Replace a subsystem by another implementation for the
 					/// consumable message type.
@@ -301,6 +310,28 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream {
 		S, #( #baggage_generic_ty, )* #( #subsystem_generics, )*
 	};
 
+	let builder_where_clause = info
+		.subsystems()
+		.iter()
+		.map(|ssf| {
+			let field_type = &ssf.generic;
+			let consumes = &ssf.message_to_consume;
+			let subsystem_sender_trait = format_ident!("{}SenderTrait", ssf.generic);
+			let subsystem_ctx_trait = format_ident!("{}ContextTrait", ssf.generic);
+			quote! {
+				#field_type:
+					#support_crate::Subsystem< #subsystem_ctx_name < #consumes>, #error_ty>,
+				<#subsystem_ctx_name< #consumes > as #subsystem_ctx_trait>::Sender:
+					#subsystem_sender_trait,
+				#subsystem_ctx_name< #consumes >:
+					#subsystem_ctx_trait,
+			}
+		})
+		.fold(TokenStream::new(), |mut ts, addendum| {
+			ts.extend(addendum);
+			ts
+		});
+
 	let mut ts = quote! {
 		/// Convenience alias.
 		type SubsystemInitFn<T> = Box<dyn FnOnce(#handle) -> ::std::result::Result<T, #error_ty> >;
@@ -332,14 +363,15 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream {
 			}
 		}
 
-		impl<S, #( #baggage_generic_ty, )*> #overseer_name <S, #( #baggage_generic_ty, )*> where #spawner_where_clause {
+		impl<S #(, #baggage_generic_ty )*> #overseer_name <S #(, #baggage_generic_ty)*>
+		where
+			#spawner_where_clause,
+		{
 			/// Create a new overseer utilizing the builder.
 			pub fn builder< #( #subsystem_generics),* >() ->
-				#builder<Missing<S> #(, Missing<#field_type> )* >
+				#builder<Missing<S> #(, Missing< #field_type > )* >
 			where
-				#(
-					#subsystem_generics : Subsystem<#subsystem_ctx_name< #consumes >, #error_ty>,
-				)*
+				#builder_where_clause
 			{
 				#builder :: new()
 			}
@@ -398,7 +430,8 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream {
 
 	ts.extend(quote!{
 		/// Builder pattern to create compile time safe construction path.
-		pub struct #builder <InitStateSpawner, #( #subsystem_passthrough_state_generics, )* #( #baggage_passthrough_state_generics, )*> {
+		pub struct #builder <InitStateSpawner, #( #subsystem_passthrough_state_generics, )* #( #baggage_passthrough_state_generics, )*>
+		{
 			#(
 				#subsystem_name: #subsystem_passthrough_state_generics,
 			)*
@@ -445,7 +478,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream {
 		impl<S, #( #subsystem_passthrough_state_generics, )* #( #baggage_passthrough_state_generics, )*>
 			#builder<Missing<S>, #( #subsystem_passthrough_state_generics, )* #( #baggage_passthrough_state_generics, )*>
 		where
-			#spawner_where_clause
+			#spawner_where_clause,
 		{
 			/// The `spawner` to use for spawning tasks.
 			pub fn spawner(self, spawner: S) -> #builder<
@@ -490,6 +523,12 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream {
 		}
 	});
 
+	// Create the string literals for spawn.
+	let subsystem_name_str_literal = subsystem_name
+		.iter()
+		.map(|ident| proc_macro2::Literal::string(ident.to_string().replace("_", "-").as_str()))
+		.collect::<Vec<_>>();
+
 	ts.extend(quote! {
 		/// Type used to represent a builder where all fields are initialized and the overseer could be constructed.
 		pub type #initialized_builder<#initialized_builder_generics> = #builder<Init<S>, #( Init<#field_type>, )*>;
@@ -498,9 +537,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream {
 		impl<#initialized_builder_generics> #initialized_builder<#initialized_builder_generics>
 		where
 			#spawner_where_clause,
-			#(
-				#subsystem_generics : Subsystem<#subsystem_ctx_name< #consumes >, #error_ty>,
-			)*
+			#builder_where_clause
 		{
 			/// Complete the construction and create the overseer type.
 			pub fn build(self)
@@ -577,17 +614,12 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream {
 						self.signal_capacity.unwrap_or(SIGNAL_CHANNEL_CAPACITY)
 					);
 
-					// Generate subsystem name based on overseer field name.
-					let subsystem_string = String::from(stringify!(#subsystem_name));
-					// Convert owned `snake case` string to a `kebab case` static str.
-					let subsystem_static_str = Box::leak(subsystem_string.replace("_", "-").into_boxed_str());
-
 					let ctx = #subsystem_ctx_name::< #consumes >::new(
 						signal_rx,
 						message_rx,
 						channels_out.clone(),
 						to_overseer_tx.clone(),
-						subsystem_static_str
+						#subsystem_name_str_literal
 					);
 
 					let #subsystem_name: OverseenSubsystem< #consumes > =
@@ -598,7 +630,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream {
 							unbounded_meter,
 							ctx,
 							#subsystem_name,
-							subsystem_static_str,
+							#subsystem_name_str_literal,
 							&mut running_subsystems,
 						)?;
 				)*

polkadot/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs

@@ -65,6 +65,7 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result<proc_macro
 				signals_received: usize,
 				message: #message_wrapper,
 			) {
+
 				let res: ::std::result::Result<_, _> = match message {
 				#(
 					#message_wrapper :: #consumes_variant ( inner ) => {
@@ -79,6 +80,13 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result<proc_macro
 				)*
 					// dummy message type
 					#message_wrapper :: Empty => Ok(()),
+
+					#[allow(unreachable_patterns)]
+					// And everything that's not WIP but no subsystem consumes it
+					unused_msg => {
+						#support_crate :: gum :: warn!("Nothing consumes {:?}", unused_msg);
+						Ok(())
+					}
 				};
 
 				if let Err(subsystem_name) = res {
@@ -110,7 +118,14 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result<proc_macro
 					#message_wrapper :: #unconsumes_variant ( _ ) => Ok(()),
 				)*
 					// dummy message type
-					#message_wrapper :: Empty => Ok(())
+					#message_wrapper :: Empty => Ok(()),
+
+					// And everything that's not WIP but no subsystem consumes it
+					#[allow(unreachable_patterns)]
+					unused_msg => {
+						#support_crate :: gum :: warn!("Nothing consumes {:?}", unused_msg);
+						Ok(())
+					}
 				};
 
 				if let Err(subsystem_name) = res {

polkadot/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs

@@ -1,70 +0,0 @@
-// Copyright 2021 Parity Technologies (UK) Ltd.
-// This file is part of Polkadot.
-
-// Polkadot is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Polkadot is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
-
-use super::*;
-use proc_macro2::{Ident, TokenStream};
-use quote::quote;
-use syn::Path;
-
-pub(crate) fn impl_dispatch(info: &OverseerInfo) -> TokenStream {
-	let message_wrapper = &info.message_wrapper;
-
-	let dispatchable_variant = info
-		.subsystems()
-		.into_iter()
-		.filter(|ssf| !ssf.no_dispatch)
-		.filter(|ssf| !ssf.wip)
-		.map(|ssf| ssf.generic.clone())
-		.collect::<Vec<Ident>>();
-
-	let dispatchable_message = info
-		.subsystems()
-		.into_iter()
-		.filter(|ssf| !ssf.no_dispatch)
-		.filter(|ssf| !ssf.wip)
-		.map(|ssf| ssf.consumes.clone())
-		.collect::<Vec<Path>>();
-
-	let mut ts = TokenStream::new();
-	if let Some(extern_network_ty) = &info.extern_network_ty.clone() {
-		ts.extend(quote! {
-			impl #message_wrapper {
-				/// Generated dispatch iterator generator.
-				pub fn dispatch_iter(extern_msg: #extern_network_ty) -> impl Iterator<Item=Self> + Send {
-					[
-					#(
-						extern_msg
-							// focuses on a `NetworkBridgeEvent< protocol_v1::* >`
-							// TODO do not require this to be hardcoded, either externalize or ...
-							// https://github.com/paritytech/polkadot/issues/3427
-							.focus()
-							.ok()
-							.map(|event| {
-								#message_wrapper :: #dispatchable_variant (
-									// the inner type of the enum variant
-									#dispatchable_message :: from( event )
-								)
-							}),
-					)*
-					]
-					.into_iter()
-					.filter_map(|x: Option<_>| x)
-				}
-			}
-		});
-	}
-	ts
-}

polkadot/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs

@@ -21,7 +21,7 @@ use super::*;
 
 /// Generates the wrapper type enum.
 pub(crate) fn impl_message_wrapper_enum(info: &OverseerInfo) -> Result<proc_macro2::TokenStream> {
-	let consumes = info.consumes();
+	let consumes = info.any_message();
 	let consumes_variant = info.variant_names();
 
 	let outgoing = &info.outgoing_ty;
@@ -52,7 +52,8 @@ pub(crate) fn impl_message_wrapper_enum(info: &OverseerInfo) -> Result<proc_macr
 	};
 
 	let ts = quote! {
-		/// Generated message type wrapper
+		/// Generated message type wrapper over all possible messages
+		/// used by any subsystem.
 		#[allow(missing_docs)]
 		#[derive(Debug)]
 		pub enum #message_wrapper {

polkadot/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs

@@ -1,259 +0,0 @@
-// Copyright 2021 Parity Technologies (UK) Ltd.
-// This file is part of Polkadot.
-
-// Polkadot is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Polkadot is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
-
-use quote::quote;
-use syn::Ident;
-
-use super::*;
-
-/// Implement a builder pattern for the `Overseer`-type,
-/// which acts as the gateway to constructing the overseer.
-pub(crate) fn impl_misc(info: &OverseerInfo) -> proc_macro2::TokenStream {
-	let overseer_name = info.overseer_name.clone();
-	let subsystem_sender_name =
-		Ident::new(&(overseer_name.to_string() + "SubsystemSender"), overseer_name.span());
-	let subsystem_ctx_name =
-		Ident::new(&(overseer_name.to_string() + "SubsystemContext"), overseer_name.span());
-	let consumes = &info.consumes();
-	let signal = &info.extern_signal_ty;
-	let wrapper_message = &info.message_wrapper;
-	let error_ty = &info.extern_error_ty;
-	let support_crate = info.support_crate_name();
-
-	let ts = quote! {
-		/// Connector to send messages towards all subsystems,
-		/// while tracking the which signals where already received.
-		#[derive(Debug, Clone)]
-		pub struct #subsystem_sender_name {
-			/// Collection of channels to all subsystems.
-			channels: ChannelsOut,
-			/// Systemwide tick for which signals were received by all subsystems.
-			signals_received: SignalsReceived,
-		}
-
-		/// implementation for wrapping message type...
-		#[#support_crate ::async_trait]
-		impl SubsystemSender< #wrapper_message > for #subsystem_sender_name {
-			async fn send_message(&mut self, msg: #wrapper_message) {
-				self.channels.send_and_log_error(self.signals_received.load(), msg).await;
-			}
-
-			async fn send_messages<T>(&mut self, msgs: T)
-			where
-				T: IntoIterator<Item = #wrapper_message> + Send,
-				T::IntoIter: Send,
-			{
-				// This can definitely be optimized if necessary.
-				for msg in msgs {
-					self.send_message(msg).await;
-				}
-			}
-
-			fn send_unbounded_message(&mut self, msg: #wrapper_message) {
-				self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg);
-			}
-		}
-
-		// ... but also implement for all individual messages to avoid
-		// the necessity for manual wrapping, and do the conversion
-		// based on the generated `From::from` impl for the individual variants.
-		#(
-		#[#support_crate ::async_trait]
-		impl SubsystemSender< #consumes > for #subsystem_sender_name {
-			async fn send_message(&mut self, msg: #consumes) {
-				self.channels.send_and_log_error(self.signals_received.load(), #wrapper_message ::from ( msg )).await;
-			}
-
-			async fn send_messages<T>(&mut self, msgs: T)
-			where
-				T: IntoIterator<Item = #consumes> + Send,
-				T::IntoIter: Send,
-			{
-				// This can definitely be optimized if necessary.
-				for msg in msgs {
-					self.send_message(msg).await;
-				}
-			}
-
-			fn send_unbounded_message(&mut self, msg: #consumes) {
-				self.channels.send_unbounded_and_log_error(self.signals_received.load(), #wrapper_message ::from ( msg ));
-			}
-		}
-		)*
-
-		/// A context type that is given to the [`Subsystem`] upon spawning.
-		/// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s
-		/// or to spawn it's [`SubsystemJob`]s.
-		///
-		/// [`Overseer`]: struct.Overseer.html
-		/// [`Subsystem`]: trait.Subsystem.html
-		/// [`SubsystemJob`]: trait.SubsystemJob.html
-		#[derive(Debug)]
-		#[allow(missing_docs)]
-		pub struct #subsystem_ctx_name<M>{
-			signals: #support_crate ::metered::MeteredReceiver< #signal >,
-			messages: SubsystemIncomingMessages<M>,
-			to_subsystems: #subsystem_sender_name,
-			to_overseer: #support_crate ::metered::UnboundedMeteredSender<
-				#support_crate ::ToOverseer
-				>,
-			signals_received: SignalsReceived,
-			pending_incoming: Option<(usize, M)>,
-			name: &'static str
-		}
-
-		impl<M> #subsystem_ctx_name<M> {
-			/// Create a new context.
-			fn new(
-				signals: #support_crate ::metered::MeteredReceiver< #signal >,
-				messages: SubsystemIncomingMessages<M>,
-				to_subsystems: ChannelsOut,
-				to_overseer: #support_crate ::metered::UnboundedMeteredSender<#support_crate:: ToOverseer>,
-				name: &'static str
-			) -> Self {
-				let signals_received = SignalsReceived::default();
-				#subsystem_ctx_name {
-					signals,
-					messages,
-					to_subsystems: #subsystem_sender_name {
-						channels: to_subsystems,
-						signals_received: signals_received.clone(),
-					},
-					to_overseer,
-					signals_received,
-					pending_incoming: None,
-					name
-				}
-			}
-
-			fn name(&self) -> &'static str {
-				self.name
-			}
-		}
-
-		#[#support_crate ::async_trait]
-		impl<M: std::fmt::Debug + Send + 'static> #support_crate ::SubsystemContext for #subsystem_ctx_name<M>
-		where
-			#subsystem_sender_name: #support_crate ::SubsystemSender< #wrapper_message >,
-			#wrapper_message: From<M>,
-		{
-			type Message = M;
-			type Signal = #signal;
-			type Sender = #subsystem_sender_name;
-			type AllMessages = #wrapper_message;
-			type Error = #error_ty;
-
-			async fn try_recv(&mut self) -> ::std::result::Result<Option<FromOverseer<M, #signal>>, ()> {
-				match #support_crate ::poll!(self.recv()) {
-					#support_crate ::Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)),
-					#support_crate ::Poll::Pending => Ok(None),
-				}
-			}
-
-			async fn recv(&mut self) -> ::std::result::Result<FromOverseer<M, #signal>, #error_ty> {
-				loop {
-					// If we have a message pending an overseer signal, we only poll for signals
-					// in the meantime.
-					if let Some((needs_signals_received, msg)) = self.pending_incoming.take() {
-						if needs_signals_received <= self.signals_received.load() {
-							return Ok(#support_crate ::FromOverseer::Communication { msg });
-						} else {
-							self.pending_incoming = Some((needs_signals_received, msg));
-
-							// wait for next signal.
-							let signal = self.signals.next().await
-								.ok_or(#support_crate ::OverseerError::Context(
-									"Signal channel is terminated and empty."
-									.to_owned()
-								))?;
-
-							self.signals_received.inc();
-							return Ok(#support_crate ::FromOverseer::Signal(signal))
-						}
-					}
-
-					let mut await_message = self.messages.next().fuse();
-					let mut await_signal = self.signals.next().fuse();
-					let signals_received = self.signals_received.load();
-					let pending_incoming = &mut self.pending_incoming;
-
-					// Otherwise, wait for the next signal or incoming message.
-					let from_overseer = #support_crate ::futures::select_biased! {
-						signal = await_signal => {
-							let signal = signal
-								.ok_or(#support_crate ::OverseerError::Context(
-									"Signal channel is terminated and empty."
-									.to_owned()
-								))?;
-
-							#support_crate ::FromOverseer::Signal(signal)
-						}
-						msg = await_message => {
-							let packet = msg
-								.ok_or(#support_crate ::OverseerError::Context(
-									"Message channel is terminated and empty."
-									.to_owned()
-								))?;
-
-							if packet.signals_received > signals_received {
-								// wait until we've received enough signals to return this message.
-								*pending_incoming = Some((packet.signals_received, packet.message));
-								continue;
-							} else {
-								// we know enough to return this message.
-								#support_crate ::FromOverseer::Communication { msg: packet.message}
-							}
-						}
-					};
-
-					if let #support_crate ::FromOverseer::Signal(_) = from_overseer {
-						self.signals_received.inc();
-					}
-
-					return Ok(from_overseer);
-				}
-			}
-
-			fn sender(&mut self) -> &mut Self::Sender {
-				&mut self.to_subsystems
-			}
-
-			fn spawn(&mut self, name: &'static str, s: Pin<Box<dyn Future<Output = ()> + Send>>)
-				-> ::std::result::Result<(), #error_ty>
-			{
-				self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnJob {
-					name,
-					subsystem: Some(self.name()),
-					s,
-				}).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?;
-				Ok(())
-			}
-
-			fn spawn_blocking(&mut self, name: &'static str, s: Pin<Box<dyn Future<Output = ()> + Send>>)
-				-> ::std::result::Result<(), #error_ty>
-			{
-				self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnBlockingJob {
-					name,
-					subsystem: Some(self.name()),
-					s,
-				}).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?;
-				Ok(())
-			}
-		}
-	};
-
-	ts
-}

polkadot/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs

@@ -37,7 +37,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> proc_macro2::TokenStr
 			S: #support_crate ::SpawnNamed,
 	};
 	// TODO add `where ..` clauses for baggage types
-	// TODO https://github.com/paritytech/polkadot/issues/3427
+	// TODO <https://github.com/paritytech/polkadot/issues/3427>
 
 	let consumes = &info.consumes_without_wip();
 	let consumes_variant = &info.variant_names_without_wip();
@@ -148,6 +148,12 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> proc_macro2::TokenStr
 						#message_wrapper :: #unconsumes_variant ( _ ) => {}
 					)*
 					#message_wrapper :: Empty => {}
+
+					// And everything that's not WIP but no subsystem consumes it
+					#[allow(unreachable_patterns)]
+					unused_msg => {
+						#support_crate :: gum :: warn!("Nothing consumes {:?}", unused_msg);
+					}
 				}
 				Ok(())
 			}

polkadot/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs

@@ -0,0 +1,712 @@
+// Copyright 2022 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use proc_macro2::TokenStream;
+use quote::quote;
+use syn::{Ident, Path, Result, Type};
+
+use petgraph::{
+	dot::{self, Dot},
+	graph::NodeIndex,
+	visit::EdgeRef,
+	Direction,
+};
+use std::collections::HashMap;
+
+use super::*;
+
+/// Render a graphviz (aka dot graph) to a file.
+fn graphviz(
+	graph: &petgraph::Graph<Ident, Path>,
+	dest: &mut impl std::io::Write,
+) -> std::io::Result<()> {
+	let config = &[dot::Config::EdgeNoLabel, dot::Config::NodeNoLabel][..];
+	let dot = Dot::with_attr_getters(
+		graph,
+		config,
+		&|_graph, edge| -> String {
+			format!(
+				r#"label="{}""#,
+				edge.weight().get_ident().expect("Must have a trailing identifier. qed")
+			)
+		},
+		&|_graph, (_node_index, subsystem_name)| -> String {
+			format!(r#"label="{}""#, subsystem_name,)
+		},
+	);
+	dest.write_all(format!("{:?}", &dot).as_bytes())?;
+	Ok(())
+}
+
+/// Generates all subsystem types and related accumulation traits.
+pub(crate) fn impl_subsystem_types_all(info: &OverseerInfo) -> Result<TokenStream> {
+	let mut ts = TokenStream::new();
+
+	let overseer_name = &info.overseer_name;
+	let span = overseer_name.span();
+	let all_messages_wrapper = &info.message_wrapper;
+	let support_crate = info.support_crate_name();
+	let signal_ty = &info.extern_signal_ty;
+	let error_ty = &info.extern_error_ty;
+
+	// create a directed graph with all the subsystems as nodes and the messages as edges
+	// key is always the message path, values are node indices in the graph and the subsystem generic identifier
+	// store the message path and the source sender, both in the graph as well as identifier
+	let mut outgoing_lut = HashMap::<&Path, Vec<(Ident, NodeIndex)>>::with_capacity(128);
+	// same for consuming the incoming messages
+	let mut consuming_lut = HashMap::<&Path, (Ident, NodeIndex)>::with_capacity(128);
+
+	// Ident = Node = subsystem generic names
+	// Path = Edge = messages
+	let mut graph = petgraph::Graph::<Ident, Path>::new();
+
+	// prepare the full index of outgoing and source subsystems
+	for ssf in info.subsystems() {
+		let node_index = graph.add_node(ssf.generic.clone());
+		for outgoing in ssf.messages_to_send.iter() {
+			outgoing_lut
+				.entry(outgoing)
+				.or_default()
+				.push((ssf.generic.clone(), node_index));
+		}
+		consuming_lut.insert(&ssf.message_to_consume, (ssf.generic.clone(), node_index));
+	}
+
+	for (message_ty, (_consuming_subsystem_ident, consuming_node_index)) in consuming_lut.iter() {
+		// match the outgoing ones that were registered above with the consumed message
+		if let Some(origin_subsystems) = outgoing_lut.get(message_ty) {
+			for (_origin_subsystem_ident, sending_node_index) in origin_subsystems.iter() {
+				graph.add_edge(*sending_node_index, *consuming_node_index, (*message_ty).clone());
+			}
+		}
+	}
+
+	// All outgoing edges are now usable to derive everything we need
+	for node_index in graph.node_indices() {
+		let subsystem_name = graph[node_index].to_string();
+		let outgoing_wrapper = Ident::new(&(subsystem_name + "OutgoingMessages"), span);
+
+		// cannot be a hashmap, duplicate keys and sorting required
+		// maps outgoing messages to the subsystem that consumes it
+		let outgoing_to_consumer = graph
+			.edges_directed(node_index, Direction::Outgoing)
+			.map(|edge| {
+				let message_ty = edge.weight();
+				let subsystem_generic_consumer = graph[edge.target()].clone();
+				Ok((to_variant(message_ty, span.clone())?, subsystem_generic_consumer))
+			})
+			.collect::<Result<Vec<(Ident, Ident)>>>()?;
+
+		// Split it for usage with quote
+		let outgoing_variant = outgoing_to_consumer.iter().map(|x| x.0.clone()).collect::<Vec<_>>();
+		let subsystem_generic = outgoing_to_consumer.into_iter().map(|x| x.1).collect::<Vec<_>>();
+
+		ts.extend(quote! {
+			impl ::std::convert::From< #outgoing_wrapper > for #all_messages_wrapper {
+				fn from(message: #outgoing_wrapper) -> Self {
+					match message {
+					#(
+						#outgoing_wrapper :: #outgoing_variant ( msg ) => #all_messages_wrapper :: #subsystem_generic ( msg ),
+					)*
+						#outgoing_wrapper :: Empty => #all_messages_wrapper :: Empty,
+						// And everything that's not WIP but no subsystem consumes it
+						#[allow(unreachable_patterns)]
+						unused_msg => {
+							#support_crate :: gum :: warn!("Nothing consumes {:?}", unused_msg);
+							#all_messages_wrapper :: Empty
+						}
+					}
+				}
+			}
+		})
+	}
+
+	// Dump the graph to file.
+	if cfg!(feature = "graph") || true {
+		let path = std::path::PathBuf::from(env!("OUT_DIR"))
+			.join(overseer_name.to_string().to_lowercase() + "-subsystem-messaging.dot");
+		if let Err(e) = std::fs::OpenOptions::new()
+			.truncate(true)
+			.create(true)
+			.write(true)
+			.open(&path)
+			.and_then(|mut f| graphviz(&graph, &mut f))
+		{
+			eprintln!("Failed to write dot graph to {}: {:?}", path.display(), e);
+		} else {
+			println!("Wrote dot graph to {}", path.display());
+		}
+	}
+
+	let subsystem_sender_name = &Ident::new(&(overseer_name.to_string() + "Sender"), span);
+	let subsystem_ctx_name = &Ident::new(&(overseer_name.to_string() + "SubsystemContext"), span);
+	ts.extend(impl_subsystem_context(info, &subsystem_sender_name, &subsystem_ctx_name));
+
+	ts.extend(impl_associate_outgoing_messages_trait(&all_messages_wrapper));
+
+	ts.extend(impl_subsystem_sender(
+		support_crate,
+		info.subsystems().iter().map(|ssf| {
+			let outgoing_wrapper =
+				Ident::new(&(ssf.generic.to_string() + "OutgoingMessages"), span);
+			outgoing_wrapper
+		}),
+		&all_messages_wrapper,
+		&subsystem_sender_name,
+	));
+
+	// Create all subsystem specific types, one by one
+	for ssf in info.subsystems() {
+		let subsystem_name = ssf.generic.to_string();
+		let outgoing_wrapper = &Ident::new(&(subsystem_name.clone() + "OutgoingMessages"), span);
+
+		let subsystem_ctx_trait = &Ident::new(&(subsystem_name.clone() + "ContextTrait"), span);
+		let subsystem_sender_trait = &Ident::new(&(subsystem_name.clone() + "SenderTrait"), span);
+
+		ts.extend(impl_per_subsystem_helper_traits(
+			info,
+			subsystem_ctx_name,
+			subsystem_ctx_trait,
+			subsystem_sender_name,
+			subsystem_sender_trait,
+			&ssf.message_to_consume,
+			&ssf.messages_to_send,
+			outgoing_wrapper,
+		));
+
+		ts.extend(impl_associate_outgoing_messages(&ssf.message_to_consume, &outgoing_wrapper));
+
+		ts.extend(impl_wrapper_enum(&outgoing_wrapper, ssf.messages_to_send.as_slice())?);
+	}
+
+	// impl the emtpy tuple handling for tests
+	let empty_tuple: Type = parse_quote! { () };
+	ts.extend(impl_subsystem_context_trait_for(
+		empty_tuple.clone(),
+		&[],
+		empty_tuple,
+		all_messages_wrapper,
+		subsystem_ctx_name,
+		subsystem_sender_name,
+		support_crate,
+		signal_ty,
+		error_ty,
+	));
+
+	Ok(ts)
+}
+
+/// Extract the final component of the message type path as used in the `#[subsystem(consumes: path::to::Foo)]` annotation.
+fn to_variant(path: &Path, span: Span) -> Result<Ident> {
+	let ident = path
+		.segments
+		.last()
+		.ok_or_else(|| syn::Error::new(span, "Path is empty, but it must end with an identifier"))
+		.map(|segment| segment.ident.clone())?;
+	Ok(ident)
+}
+
+/// Converts the outgoing message types to variants.
+///
+/// Note: Commonly this is `${X}Message` becomes `${X}OutgoingMessages::${X}Message`
+/// where for `AllMessages` it would be `AllMessages::${X}`.
+fn to_variants(message_types: &[Path], span: Span) -> Result<Vec<Ident>> {
+	let variants: Vec<_> =
+		Result::from_iter(message_types.into_iter().map(|path| to_variant(path, span.clone())))?;
+	Ok(variants)
+}
+
+/// Generates the wrapper type enum, no bells or whistles.
+pub(crate) fn impl_wrapper_enum(wrapper: &Ident, message_types: &[Path]) -> Result<TokenStream> {
+	// The message types are path based, each of them must finish with a type
+	// and as such we do this upfront.
+	let variants = to_variants(message_types, wrapper.span())?;
+
+	let ts = quote! {
+		#[allow(missing_docs)]
+		#[derive(Debug)]
+		pub enum #wrapper {
+			#(
+				#variants ( #message_types ),
+			)*
+			Empty,
+		}
+
+		#(
+			impl ::std::convert::From< #message_types > for #wrapper {
+				fn from(message: #message_types) -> Self {
+					#wrapper :: #variants ( message )
+				}
+			}
+		)*
+
+		// Useful for unit and integration tests:
+		impl ::std::convert::From< () > for #wrapper {
+			fn from(_message: ()) -> Self {
+				#wrapper :: Empty
+			}
+		}
+	};
+	Ok(ts)
+}
+
+/// Create the subsystem sender type and implements `trait SubsystemSender`
+/// for the `#outgoing_wrappers: From<OutgoingMessage>` with the proper associated types.
+pub(crate) fn impl_subsystem_sender(
+	support_crate: &Path,
+	outgoing_wrappers: impl IntoIterator<Item = Ident>,
+	all_messages_wrapper: &Ident,
+	subsystem_sender_name: &Ident,
+) -> TokenStream {
+	let mut ts = quote! {
+		/// Connector to send messages towards all subsystems,
+		/// while tracking the which signals where already received.
+		#[derive(Debug)]
+		pub struct #subsystem_sender_name < OutgoingWrapper > {
+			/// Collection of channels to all subsystems.
+			channels: ChannelsOut,
+			/// Systemwide tick for which signals were received by all subsystems.
+			signals_received: SignalsReceived,
+			/// Keep that marker around.
+			_phantom: ::core::marker::PhantomData< OutgoingWrapper >,
+		}
+
+		// can't derive due to `PhantomData` and `OutgoingWrapper` not being
+		// bounded by `Clone`.
+		impl<OutgoingWrapper> std::clone::Clone for #subsystem_sender_name < OutgoingWrapper > {
+			fn clone(&self) -> Self {
+				Self {
+					channels: self.channels.clone(),
+					signals_received: self.signals_received.clone(),
+					_phantom: ::core::marker::PhantomData::default(),
+				}
+			}
+		}
+	};
+
+	// Create the same for a wrapping enum:
+	//
+	// 1. subsystem specific `*OutgoingMessages`-type
+	// 2. overseer-global-`AllMessages`-type
+	let wrapped = |outgoing_wrapper: &TokenStream| {
+		quote! {
+			#[#support_crate ::async_trait]
+			impl<OutgoingMessage> SubsystemSender< OutgoingMessage > for #subsystem_sender_name < #outgoing_wrapper >
+			where
+				OutgoingMessage: Send + 'static,
+				#outgoing_wrapper: ::std::convert::From<OutgoingMessage> + Send,
+				#all_messages_wrapper: ::std::convert::From< #outgoing_wrapper > + Send,
+			{
+				async fn send_message(&mut self, msg: OutgoingMessage)
+				{
+					self.channels.send_and_log_error(
+						self.signals_received.load(),
+						<#all_messages_wrapper as ::std::convert::From<_>> ::from (
+							<#outgoing_wrapper as ::std::convert::From<_>> :: from ( msg )
+						)
+					).await;
+				}
+
+				async fn send_messages<I>(&mut self, msgs: I)
+				where
+					I: IntoIterator<Item=OutgoingMessage> + Send,
+					I::IntoIter: Iterator<Item=OutgoingMessage> + Send,
+				{
+					for msg in msgs {
+						self.send_message( msg ).await;
+					}
+				}
+
+				fn send_unbounded_message(&mut self, msg: OutgoingMessage)
+				{
+					self.channels.send_unbounded_and_log_error(
+						self.signals_received.load(),
+						<#all_messages_wrapper as ::std::convert::From<_>> ::from (
+							<#outgoing_wrapper as ::std::convert::From<_>> :: from ( msg )
+						)
+					);
+				}
+			}
+		}
+	};
+
+	for outgoing_wrapper in outgoing_wrappers {
+		ts.extend(wrapped(&quote! {
+			#outgoing_wrapper
+		}));
+	}
+
+	ts.extend(wrapped(&quote! {
+		()
+	}));
+
+	ts
+}
+
+/// Define the `trait AssociateOutgoing` and implement it for  `#all_messages_wrapper` and `()`.
+pub(crate) fn impl_associate_outgoing_messages_trait(all_messages_wrapper: &Ident) -> TokenStream {
+	quote! {
+		/// Binds a generated type covering all declared outgoing messages,
+		/// which implements `#generated_outgoing: From<M>` for all annotated types
+		/// of a particular subsystem.
+		///
+		/// Note: This works because there is a 1?:1 relation between consumed messages and subsystems.
+		pub trait AssociateOutgoing: ::std::fmt::Debug + Send {
+			/// The associated _outgoing_ messages for a subsystem that _consumes_ the message `Self`.
+			type OutgoingMessages: Into< #all_messages_wrapper > + ::std::fmt::Debug + Send;
+		}
+
+		// Helper for tests, where nothing is ever sent.
+		impl AssociateOutgoing for () {
+			type OutgoingMessages = ();
+		}
+
+		// Helper for tests, allows sending of arbitrary messages give
+		// an test context.
+		impl AssociateOutgoing for #all_messages_wrapper {
+			type OutgoingMessages = #all_messages_wrapper ;
+		}
+	}
+}
+
+/// Implement `AssociateOutgoing` for `#consumes` being handled by a particular subsystem.
+///
+/// Binds the outgoing messages to the inbound message.
+///
+/// Note: Works, since there is a 1:1 relation between inbound message type and subsystem declarations.
+/// Note: A workaround until default associated types work in `rustc`.
+pub(crate) fn impl_associate_outgoing_messages(
+	consumes: &Path,
+	outgoing_wrapper: &Ident,
+) -> TokenStream {
+	quote! {
+		impl AssociateOutgoing for #outgoing_wrapper {
+			type OutgoingMessages = #outgoing_wrapper;
+		}
+
+		impl AssociateOutgoing for #consumes {
+			type OutgoingMessages = #outgoing_wrapper;
+		}
+	}
+}
+
+/// Implement `trait SubsystemContext` for a particular subsystem context,
+/// that is generated by the proc-macro too.
+pub(crate) fn impl_subsystem_context_trait_for(
+	consumes: Type,
+	outgoing: &[Type],
+	outgoing_wrapper: Type,
+	all_messages_wrapper: &Ident,
+	subsystem_ctx_name: &Ident,
+	subsystem_sender_name: &Ident,
+	support_crate: &Path,
+	signal: &Path,
+	error_ty: &Path,
+) -> TokenStream {
+	// impl the subsystem context trait
+	let where_clause = quote! {
+		#consumes: AssociateOutgoing + ::std::fmt::Debug + Send + 'static,
+		#all_messages_wrapper: From< #outgoing_wrapper >,
+		#all_messages_wrapper: From< #consumes >,
+		#outgoing_wrapper: #( From< #outgoing > )+*,
+	};
+
+	quote! {
+		#[#support_crate ::async_trait]
+		impl #support_crate ::SubsystemContext for #subsystem_ctx_name < #consumes >
+		where
+			#where_clause
+		{
+			type Message = #consumes;
+			type Signal = #signal;
+			type OutgoingMessages = #outgoing_wrapper;
+			type Sender = #subsystem_sender_name < #outgoing_wrapper >;
+			type Error = #error_ty;
+
+			async fn try_recv(&mut self) -> ::std::result::Result<Option<FromOverseer< Self::Message, #signal>>, ()> {
+				match #support_crate ::poll!(self.recv()) {
+					#support_crate ::Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)),
+					#support_crate ::Poll::Pending => Ok(None),
+				}
+			}
+
+			async fn recv(&mut self) -> ::std::result::Result<FromOverseer<Self::Message, #signal>, #error_ty> {
+				loop {
+					// If we have a message pending an overseer signal, we only poll for signals
+					// in the meantime.
+					if let Some((needs_signals_received, msg)) = self.pending_incoming.take() {
+						if needs_signals_received <= self.signals_received.load() {
+							return Ok( #support_crate ::FromOverseer::Communication { msg });
+						} else {
+							self.pending_incoming = Some((needs_signals_received, msg));
+
+							// wait for next signal.
+							let signal = self.signals.next().await
+								.ok_or(#support_crate ::OverseerError::Context(
+									"Signal channel is terminated and empty."
+									.to_owned()
+								))?;
+
+							self.signals_received.inc();
+							return Ok( #support_crate ::FromOverseer::Signal(signal))
+						}
+					}
+
+					let mut await_message = self.messages.next().fuse();
+					let mut await_signal = self.signals.next().fuse();
+					let signals_received = self.signals_received.load();
+					let pending_incoming = &mut self.pending_incoming;
+
+					// Otherwise, wait for the next signal or incoming message.
+					let from_overseer = #support_crate ::futures::select_biased! {
+						signal = await_signal => {
+							let signal = signal
+								.ok_or( #support_crate ::OverseerError::Context(
+									"Signal channel is terminated and empty."
+									.to_owned()
+								))?;
+
+							#support_crate ::FromOverseer::Signal(signal)
+						}
+						msg = await_message => {
+							let packet = msg
+								.ok_or( #support_crate ::OverseerError::Context(
+									"Message channel is terminated and empty."
+									.to_owned()
+								))?;
+
+							if packet.signals_received > signals_received {
+								// wait until we've received enough signals to return this message.
+								*pending_incoming = Some((packet.signals_received, packet.message));
+								continue;
+							} else {
+								// we know enough to return this message.
+								#support_crate ::FromOverseer::Communication { msg: packet.message}
+							}
+						}
+					};
+
+					if let #support_crate ::FromOverseer::Signal(_) = from_overseer {
+						self.signals_received.inc();
+					}
+
+					return Ok(from_overseer);
+				}
+			}
+
+			fn sender(&mut self) -> &mut Self::Sender {
+				&mut self.to_subsystems
+			}
+
+			fn spawn(&mut self, name: &'static str, s: Pin<Box<dyn Future<Output = ()> + Send>>)
+				-> ::std::result::Result<(), #error_ty>
+			{
+				self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnJob {
+					name,
+					subsystem: Some(self.name()),
+					s,
+				}).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?;
+				Ok(())
+			}
+
+			fn spawn_blocking(&mut self, name: &'static str, s: Pin<Box<dyn Future<Output = ()> + Send>>)
+				-> ::std::result::Result<(), #error_ty>
+			{
+				self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnBlockingJob {
+					name,
+					subsystem: Some(self.name()),
+					s,
+				}).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?;
+				Ok(())
+			}
+		}
+	}
+}
+
+/// Implement the additional subsystem accumulation traits, for simplified usage,
+/// i.e. `${Subsystem}SenderTrait` and `${Subsystem}ContextTrait`.
+pub(crate) fn impl_per_subsystem_helper_traits(
+	info: &OverseerInfo,
+	subsystem_ctx_name: &Ident,
+	subsystem_ctx_trait: &Ident,
+	subsystem_sender_name: &Ident,
+	subsystem_sender_trait: &Ident,
+	consumes: &Path,
+	outgoing: &[Path],
+	outgoing_wrapper: &Ident,
+) -> TokenStream {
+	let all_messages_wrapper = &info.message_wrapper;
+	let signal_ty = &info.extern_signal_ty;
+	let error_ty = &info.extern_error_ty;
+	let support_crate = info.support_crate_name();
+
+	let mut ts = TokenStream::new();
+
+	// Create a helper trait bound of all outgoing messages, and the generated wrapper type
+	// for ease of use within subsystems:
+	let acc_sender_trait_bounds = quote! {
+		#support_crate ::SubsystemSender< #outgoing_wrapper >
+		#(
+			+ #support_crate ::SubsystemSender< #outgoing >
+		)*
+			+ #support_crate ::SubsystemSender< () >
+			+ Send
+			+ 'static
+	};
+
+	ts.extend(quote! {
+		/// A abstracting trait for usage with subsystems.
+		pub trait #subsystem_sender_trait : #acc_sender_trait_bounds
+		{}
+
+		impl<T> #subsystem_sender_trait for T
+		where
+			T: #acc_sender_trait_bounds
+		{}
+	});
+
+	// Create a helper accumulated per subsystem trait bound:
+	let where_clause = quote! {
+		#consumes: AssociateOutgoing + ::std::fmt::Debug + Send + 'static,
+		#all_messages_wrapper: From< #outgoing_wrapper >,
+		#all_messages_wrapper: From< #consumes >,
+		#all_messages_wrapper: From< () >,
+		#outgoing_wrapper: #( From< #outgoing > )+*,
+		#outgoing_wrapper: From< () >,
+	};
+
+	ts.extend(quote! {
+		/// Accumulative trait for a particular subsystem wrapper.
+		pub trait #subsystem_ctx_trait : SubsystemContext <
+			Message = #consumes,
+			Signal = #signal_ty,
+			OutgoingMessages = #outgoing_wrapper,
+			// Sender,
+			Error = #error_ty,
+		>
+		where
+			#where_clause
+			<Self as SubsystemContext>::Sender:
+				#subsystem_sender_trait
+				+ #acc_sender_trait_bounds,
+		{
+			/// Sender.
+			type Sender: #subsystem_sender_trait;
+		}
+
+		impl<T> #subsystem_ctx_trait for T
+		where
+			T: SubsystemContext <
+				Message = #consumes,
+				Signal = #signal_ty,
+				OutgoingMessages = #outgoing_wrapper,
+				// Sender
+				Error = #error_ty,
+			>,
+			#where_clause
+			<T as SubsystemContext>::Sender:
+				#subsystem_sender_trait
+				+ #acc_sender_trait_bounds,
+		{
+			type Sender = <T as SubsystemContext>::Sender;
+		}
+	});
+
+	ts.extend(impl_subsystem_context_trait_for(
+		parse_quote! { #consumes },
+		&Vec::from_iter(outgoing.iter().map(|path| {
+			parse_quote! { #path }
+		})),
+		parse_quote! { #outgoing_wrapper },
+		all_messages_wrapper,
+		subsystem_ctx_name,
+		subsystem_sender_name,
+		support_crate,
+		signal_ty,
+		error_ty,
+	));
+	ts
+}
+
+/// Generate the subsystem context type and provide `fn new` on it.
+///
+/// Note: The generated `fn new` is used by the [builder pattern](../impl_builder.rs).
+pub(crate) fn impl_subsystem_context(
+	info: &OverseerInfo,
+	subsystem_sender_name: &Ident,
+	subsystem_ctx_name: &Ident,
+) -> TokenStream {
+	let signal_ty = &info.extern_signal_ty;
+	let support_crate = info.support_crate_name();
+
+	let ts = quote! {
+		/// A context type that is given to the [`Subsystem`] upon spawning.
+		/// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s
+		/// or to spawn it's [`SubsystemJob`]s.
+		///
+		/// [`Overseer`]: struct.Overseer.html
+		/// [`Subsystem`]: trait.Subsystem.html
+		/// [`SubsystemJob`]: trait.SubsystemJob.html
+		#[derive(Debug)]
+		#[allow(missing_docs)]
+		pub struct #subsystem_ctx_name<M: AssociateOutgoing + Send + 'static> {
+			signals: #support_crate ::metered::MeteredReceiver< #signal_ty >,
+			messages: SubsystemIncomingMessages< M >,
+			to_subsystems: #subsystem_sender_name < <M as AssociateOutgoing>::OutgoingMessages >,
+			to_overseer: #support_crate ::metered::UnboundedMeteredSender<
+				#support_crate ::ToOverseer
+				>,
+			signals_received: SignalsReceived,
+			pending_incoming: Option<(usize, M)>,
+			name: &'static str
+		}
+
+		impl<M> #subsystem_ctx_name <M>
+		where
+			M: AssociateOutgoing + Send + 'static,
+		{
+			/// Create a new context.
+			fn new(
+				signals: #support_crate ::metered::MeteredReceiver< #signal_ty >,
+				messages: SubsystemIncomingMessages< M >,
+				to_subsystems: ChannelsOut,
+				to_overseer: #support_crate ::metered::UnboundedMeteredSender<#support_crate:: ToOverseer>,
+				name: &'static str
+			) -> Self {
+				let signals_received = SignalsReceived::default();
+				#subsystem_ctx_name :: <M> {
+					signals,
+					messages,
+					to_subsystems: #subsystem_sender_name :: < <M as AssociateOutgoing>::OutgoingMessages > {
+						channels: to_subsystems,
+						signals_received: signals_received.clone(),
+						_phantom: ::core::marker::PhantomData::default(),
+					},
+					to_overseer,
+					signals_received,
+					pending_incoming: None,
+					name
+				}
+			}
+
+			fn name(&self) -> &'static str {
+				self.name
+			}
+		}
+	};
+
+	ts
+}

polkadot/node/overseer/overseer-gen/proc-macro/src/lib.rs

@@ -14,33 +14,44 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-#![deny(unused_crate_dependencies)]
-
 use proc_macro2::{Ident, Span, TokenStream};
-use quote::{quote, ToTokens};
-use syn::{parse2, Result};
+use syn::{parse_quote, spanned::Spanned, Path};
 
 mod impl_builder;
 mod impl_channels_out;
-mod impl_dispatch;
 mod impl_message_wrapper;
-mod impl_misc;
 mod impl_overseer;
-mod parse_attr;
-mod parse_struct;
-
-use impl_builder::*;
-use impl_channels_out::*;
-use impl_dispatch::*;
-use impl_message_wrapper::*;
-use impl_misc::*;
-use impl_overseer::*;
-use parse_attr::*;
-use parse_struct::*;
+mod impl_subsystem_ctx_sender;
+mod overseer;
+mod parse;
+mod subsystem;
 
 #[cfg(test)]
 mod tests;
 
+use impl_builder::*;
+use impl_channels_out::*;
+use impl_message_wrapper::*;
+use impl_overseer::*;
+use impl_subsystem_ctx_sender::*;
+use parse::*;
+
+use self::{overseer::*, subsystem::*};
+
+/// Obtain the support crate `Path` as `TokenStream`.
+pub(crate) fn support_crate() -> Result<Path, proc_macro_crate::Error> {
+	Ok(if cfg!(test) {
+		parse_quote! {crate}
+	} else {
+		use proc_macro_crate::{crate_name, FoundCrate};
+		let crate_name = crate_name("polkadot-overseer-gen")?;
+		match crate_name {
+			FoundCrate::Itself => parse_quote! {crate},
+			FoundCrate::Name(name) => Ident::new(&name, Span::call_site()).into(),
+		}
+	})
+}
+
 #[proc_macro_attribute]
 pub fn overlord(
 	attr: proc_macro::TokenStream,
@@ -53,58 +64,26 @@ pub fn overlord(
 		.into()
 }
 
-pub(crate) fn impl_overseer_gen(
-	attr: TokenStream,
-	orig: TokenStream,
-) -> Result<proc_macro2::TokenStream> {
-	let args: AttrArgs = parse2(attr)?;
-	let message_wrapper = args.message_wrapper;
-
-	let of: OverseerGuts = parse2(orig)?;
-
-	let support_crate_name = if cfg!(test) {
-		quote! {crate}
-	} else {
-		use proc_macro_crate::{crate_name, FoundCrate};
-		let crate_name = crate_name("polkadot-overseer-gen")
-			.expect("Support crate polkadot-overseer-gen is present in `Cargo.toml`. qed");
-		match crate_name {
-			FoundCrate::Itself => quote! {crate},
-			FoundCrate::Name(name) => Ident::new(&name, Span::call_site()).to_token_stream(),
-		}
-	};
-	let info = OverseerInfo {
-		support_crate_name,
-		subsystems: of.subsystems,
-		baggage: of.baggage,
-		overseer_name: of.name,
-		message_wrapper,
-		message_channel_capacity: args.message_channel_capacity,
-		signal_channel_capacity: args.signal_channel_capacity,
-		extern_event_ty: args.extern_event_ty,
-		extern_signal_ty: args.extern_signal_ty,
-		extern_error_ty: args.extern_error_ty,
-		extern_network_ty: args.extern_network_ty,
-		outgoing_ty: args.outgoing_ty,
-	};
-
-	let mut additive = impl_overseer_struct(&info);
-	additive.extend(impl_builder(&info));
-
-	additive.extend(impl_overseen_subsystem(&info));
-	additive.extend(impl_channels_out_struct(&info));
-	additive.extend(impl_misc(&info));
-
-	additive.extend(impl_message_wrapper_enum(&info)?);
-	additive.extend(impl_dispatch(&info));
-
-	let ts = expander::Expander::new("overlord-expansion")
-		.add_comment("Generated overseer code by `#[overlord(..)]`".to_owned())
-		.dry(!cfg!(feature = "expand"))
-		.verbose(false)
-		.fmt(expander::Edition::_2021)
-		.write_to_out_dir(additive)
-		.expect("Expander does not fail due to IO in OUT_DIR. qed");
-
-	Ok(ts)
+#[proc_macro_attribute]
+pub fn subsystem(
+	attr: proc_macro::TokenStream,
+	item: proc_macro::TokenStream,
+) -> proc_macro::TokenStream {
+	let attr: TokenStream = attr.into();
+	let item: TokenStream = item.into();
+	impl_subsystem_context_trait_bounds(attr, item, MakeSubsystem::ImplSubsystemTrait)
+		.unwrap_or_else(|err| err.to_compile_error())
+		.into()
+}
+
+#[proc_macro_attribute]
+pub fn contextbounds(
+	attr: proc_macro::TokenStream,
+	item: proc_macro::TokenStream,
+) -> proc_macro::TokenStream {
+	let attr: TokenStream = attr.into();
+	let item: TokenStream = item.into();
+	impl_subsystem_context_trait_bounds(attr, item, MakeSubsystem::AddContextTraitBounds)
+		.unwrap_or_else(|err| err.to_compile_error())
+		.into()
 }

polkadot/node/overseer/overseer-gen/proc-macro/src/overseer.rs

@@ -0,0 +1,67 @@
+// Copyright 2022 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use proc_macro2::TokenStream;
+use syn::{parse2, Result};
+
+use super::{parse::*, *};
+
+pub(crate) fn impl_overseer_gen(
+	attr: TokenStream,
+	orig: TokenStream,
+) -> Result<proc_macro2::TokenStream> {
+	let args: OverseerAttrArgs = parse2(attr)?;
+	let message_wrapper = args.message_wrapper;
+
+	let of: OverseerGuts = parse2(orig)?;
+
+	let support_crate = support_crate().expect("The crate this macro is run for, includes the proc-macro support as dependency, otherwise it could not be run in the first place. qed");
+	let info = OverseerInfo {
+		support_crate,
+		subsystems: of.subsystems,
+		baggage: of.baggage,
+		overseer_name: of.name,
+		message_wrapper,
+		message_channel_capacity: args.message_channel_capacity,
+		signal_channel_capacity: args.signal_channel_capacity,
+		extern_event_ty: args.extern_event_ty,
+		extern_signal_ty: args.extern_signal_ty,
+		extern_error_ty: args.extern_error_ty,
+		outgoing_ty: args.outgoing_ty,
+	};
+
+	let mut additive = impl_overseer_struct(&info);
+	additive.extend(impl_builder(&info));
+
+	additive.extend(impl_overseen_subsystem(&info));
+	additive.extend(impl_channels_out_struct(&info));
+	additive.extend(impl_subsystem_types_all(&info)?);
+
+	additive.extend(impl_message_wrapper_enum(&info)?);
+
+	let ts = expander::Expander::new("overlord-expansion")
+		.add_comment("Generated overseer code by `#[overlord(..)]`".to_owned())
+		.dry(!cfg!(feature = "expand"))
+		.verbose(true)
+		// once all our needed format options are available on stable
+		// we should enabled this again, until then too many warnings
+		// are generated
+		// .fmt(expander::Edition::_2021)
+		.write_to_out_dir(additive)
+		.expect("Expander does not fail due to IO in OUT_DIR. qed");
+
+	Ok(ts)
+}

polkadot/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs

@@ -0,0 +1,39 @@
+// Copyright 2022 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+mod kw {
+	syn::custom_keyword!(event);
+	syn::custom_keyword!(signal);
+	syn::custom_keyword!(error);
+	syn::custom_keyword!(outgoing);
+	syn::custom_keyword!(gen);
+	syn::custom_keyword!(signal_capacity);
+	syn::custom_keyword!(message_capacity);
+	syn::custom_keyword!(subsystem);
+	syn::custom_keyword!(prefix);
+}
+
+mod parse_overseer_attr;
+mod parse_overseer_struct;
+
+mod parse_subsystem_attr;
+
+#[cfg(test)]
+mod tests;
+
+pub(crate) use self::{parse_overseer_attr::*, parse_overseer_struct::*};
+
+pub(crate) use self::parse_subsystem_attr::*;

polkadot/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_attr.rs

@@ -14,6 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
+use super::kw;
 use proc_macro2::Span;
 use quote::{quote, ToTokens};
 use std::collections::{hash_map::RandomState, HashMap};
@@ -24,21 +25,9 @@ use syn::{
 	Error, Ident, LitInt, Path, Result, Token,
 };
 
-mod kw {
-	syn::custom_keyword!(event);
-	syn::custom_keyword!(signal);
-	syn::custom_keyword!(error);
-	syn::custom_keyword!(network);
-	syn::custom_keyword!(outgoing);
-	syn::custom_keyword!(gen);
-	syn::custom_keyword!(signal_capacity);
-	syn::custom_keyword!(message_capacity);
-}
-
 #[derive(Clone, Debug)]
 enum OverseerAttrItem {
 	ExternEventType { tag: kw::event, eq_token: Token![=], value: Path },
-	ExternNetworkType { tag: kw::network, eq_token: Token![=], value: Path },
 	ExternOverseerSignalType { tag: kw::signal, eq_token: Token![=], value: Path },
 	ExternErrorType { tag: kw::error, eq_token: Token![=], value: Path },
 	OutgoingType { tag: kw::outgoing, eq_token: Token![=], value: Path },
@@ -53,9 +42,6 @@ impl ToTokens for OverseerAttrItem {
 			Self::ExternEventType { tag, eq_token, value } => {
 				quote! { #tag #eq_token, #value }
 			},
-			Self::ExternNetworkType { tag, eq_token, value } => {
-				quote! { #tag #eq_token, #value }
-			},
 			Self::ExternOverseerSignalType { tag, eq_token, value } => {
 				quote! { #tag #eq_token, #value }
 			},
@@ -100,12 +86,6 @@ impl Parse for OverseerAttrItem {
 				eq_token: input.parse()?,
 				value: input.parse()?,
 			})
-		} else if lookahead.peek(kw::network) {
-			Ok(OverseerAttrItem::ExternNetworkType {
-				tag: input.parse::<kw::network>()?,
-				eq_token: input.parse()?,
-				value: input.parse()?,
-			})
 		} else if lookahead.peek(kw::outgoing) {
 			Ok(OverseerAttrItem::OutgoingType {
 				tag: input.parse::<kw::outgoing>()?,
@@ -138,15 +118,11 @@ impl Parse for OverseerAttrItem {
 
 /// Attribute arguments
 #[derive(Clone, Debug)]
-pub(crate) struct AttrArgs {
+pub(crate) struct OverseerAttrArgs {
 	pub(crate) message_wrapper: Ident,
 	pub(crate) extern_event_ty: Path,
 	pub(crate) extern_signal_ty: Path,
 	pub(crate) extern_error_ty: Path,
-	/// A external subsystem that both consumes and produces messages
-	/// but is not part of the band of subsystems, it's a mere proxy
-	/// to another entity that consumes/produces messages.
-	pub(crate) extern_network_ty: Option<Path>,
 	pub(crate) outgoing_ty: Option<Path>,
 	pub(crate) signal_channel_capacity: usize,
 	pub(crate) message_channel_capacity: usize,
@@ -170,7 +146,7 @@ macro_rules! extract_variant {
 	};
 }
 
-impl Parse for AttrArgs {
+impl Parse for OverseerAttrArgs {
 	fn parse(input: &ParseBuffer) -> Result<Self> {
 		let items: Punctuated<OverseerAttrItem, Token![,]> =
 			input.parse_terminated(OverseerAttrItem::parse)?;
@@ -198,18 +174,16 @@ impl Parse for AttrArgs {
 
 		let error = extract_variant!(unique, ExternErrorType; err = "Must declare the overseer error type via `error=..`.")?;
 		let event = extract_variant!(unique, ExternEventType; err = "Must declare the overseer event type via `event=..`.")?;
-		let signal = extract_variant!(unique, ExternOverseerSignalType; err = "Must declare the overseer signal type via `span=..`.")?;
+		let signal = extract_variant!(unique, ExternOverseerSignalType; err = "Must declare the overseer signal type via `signal=..`.")?;
 		let message_wrapper = extract_variant!(unique, MessageWrapperName; err = "Must declare the overseer generated wrapping message type via `gen=..`.")?;
-		let network = extract_variant!(unique, ExternNetworkType);
 		let outgoing = extract_variant!(unique, OutgoingType);
 
-		Ok(AttrArgs {
+		Ok(OverseerAttrArgs {
 			signal_channel_capacity,
 			message_channel_capacity,
 			extern_event_ty: event,
 			extern_signal_ty: signal,
 			extern_error_ty: error,
-			extern_network_ty: network,
 			outgoing_ty: outgoing,
 			message_wrapper,
 		})

polkadot/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs

@@ -17,32 +17,36 @@
 use proc_macro2::{Span, TokenStream};
 use std::collections::{hash_map::RandomState, HashMap, HashSet};
 use syn::{
+	parenthesized,
 	parse::{Parse, ParseStream},
 	punctuated::Punctuated,
 	spanned::Spanned,
-	AttrStyle, Attribute, Error, Field, FieldsNamed, GenericParam, Ident, ItemStruct, Path, Result,
-	Token, Type, Visibility,
+	token::Bracket,
+	AttrStyle, Error, Field, FieldsNamed, GenericParam, Ident, ItemStruct, Path, Result, Token,
+	Type, Visibility,
 };
 
 use quote::{quote, ToTokens};
 
 mod kw {
 	syn::custom_keyword!(wip);
-	syn::custom_keyword!(no_dispatch);
 	syn::custom_keyword!(blocking);
+	syn::custom_keyword!(consumes);
+	syn::custom_keyword!(sends);
 }
 
 #[derive(Clone, Debug)]
-enum SubSysAttrItem {
+pub(crate) enum SubSysAttrItem {
 	/// The subsystem is still a work in progress
 	/// and should not be communicated with.
 	Wip(kw::wip),
 	/// The subsystem is blocking and requires to be
 	/// spawned on an exclusive thread.
 	Blocking(kw::blocking),
-	/// External messages should not be - after being converted -
-	/// be dispatched to the annotated subsystem.
-	NoDispatch(kw::no_dispatch),
+	/// Message to be sent by this subsystem.
+	Sends(Sends),
+	/// Message to be consumed by this subsystem.
+	Consumes(Consumes),
 }
 
 impl Parse for SubSysAttrItem {
@@ -52,10 +56,10 @@ impl Parse for SubSysAttrItem {
 			Self::Wip(input.parse::<kw::wip>()?)
 		} else if lookahead.peek(kw::blocking) {
 			Self::Blocking(input.parse::<kw::blocking>()?)
-		} else if lookahead.peek(kw::no_dispatch) {
-			Self::NoDispatch(input.parse::<kw::no_dispatch>()?)
+		} else if lookahead.peek(kw::sends) {
+			Self::Sends(input.parse::<Sends>()?)
 		} else {
-			return Err(lookahead.error())
+			Self::Consumes(input.parse::<Consumes>()?)
 		})
 	}
 }
@@ -69,8 +73,11 @@ impl ToTokens for SubSysAttrItem {
 			Self::Blocking(blocking) => {
 				quote! { #blocking }
 			},
-			Self::NoDispatch(no_dispatch) => {
-				quote! { #no_dispatch }
+			Self::Sends(_) => {
+				quote! {}
+			},
+			Self::Consumes(_) => {
+				quote! {}
 			},
 		};
 		tokens.extend(ts.into_iter());
@@ -78,7 +85,7 @@ impl ToTokens for SubSysAttrItem {
 }
 
 /// A field of the struct annotated with
-/// `#[subsystem(no_dispatch, , A | B | C)]`
+/// `#[subsystem(A, B, C)]`
 #[derive(Clone, Debug)]
 pub(crate) struct SubSysField {
 	/// Name of the field.
@@ -87,11 +94,10 @@ pub(crate) struct SubSysField {
 	/// which is also used `#wrapper_message :: #variant` variant
 	/// part.
 	pub(crate) generic: Ident,
-	/// Type to be consumed by the subsystem.
-	pub(crate) consumes: Path,
-	/// If `no_dispatch` is present, if the message is incoming via
-	/// an `extern` `Event`, it will not be dispatched to all subsystems.
-	pub(crate) no_dispatch: bool,
+	/// Type of message to be consumed by the subsystem.
+	pub(crate) message_to_consume: Path,
+	/// Types of messages to be sent by the subsystem.
+	pub(crate) messages_to_send: Vec<Path>,
 	/// If the subsystem implementation is blocking execution and hence
 	/// has to be spawned on a separate thread or thread pool.
 	pub(crate) blocking: bool,
@@ -115,6 +121,15 @@ macro_rules! extract_variant {
 	($unique:expr, $variant:ident ; err = $err:expr) => {
 		extract_variant!($unique, $variant).ok_or_else(|| Error::new(Span::call_site(), $err))
 	};
+	($unique:expr, $variant:ident take) => {
+		$unique.values().find_map(|item| {
+			if let SubSysAttrItem::$variant(value) = item {
+				Some(value.clone())
+			} else {
+				None
+			}
+		})
+	};
 	($unique:expr, $variant:ident) => {
 		$unique.values().find_map(|item| {
 			if let SubSysAttrItem::$variant(_) = item {
@@ -126,57 +141,113 @@ macro_rules! extract_variant {
 	};
 }
 
-pub(crate) struct SubSystemTags {
+#[derive(Debug, Clone)]
+pub(crate) struct Sends {
 	#[allow(dead_code)]
-	pub(crate) attrs: Vec<Attribute>,
+	pub(crate) keyword_sends: kw::sends,
 	#[allow(dead_code)]
-	pub(crate) no_dispatch: bool,
-	/// The subsystem is in progress, only generate the `Wrapper` variant, but do not forward messages
-	/// and also not include the subsystem in the list of subsystems.
-	pub(crate) wip: bool,
-	pub(crate) blocking: bool,
+	pub(crate) colon: Token![:],
+	#[allow(dead_code)]
+	pub(crate) bracket: Option<Bracket>,
+	pub(crate) sends: Punctuated<Path, Token![,]>,
+}
+
+impl Parse for Sends {
+	fn parse(input: syn::parse::ParseStream) -> Result<Self> {
+		let content;
+		let keyword_sends = input.parse()?;
+		let colon = input.parse()?;
+		let (bracket, sends) = if !input.peek(syn::token::Bracket) {
+			let mut sends = Punctuated::new();
+			sends.push_value(input.parse::<Path>()?);
+			(None, sends)
+		} else {
+			let bracket = Some(syn::bracketed!(content in input));
+			let sends = Punctuated::parse_terminated(&content)?;
+			(bracket, sends)
+		};
+		Ok(Self { keyword_sends, colon, bracket, sends })
+	}
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct Consumes {
+	#[allow(dead_code)]
+	pub(crate) keyword_consumes: Option<kw::consumes>,
+	#[allow(dead_code)]
+	pub(crate) colon: Option<Token![:]>,
 	pub(crate) consumes: Path,
 }
 
-impl Parse for SubSystemTags {
+impl Parse for Consumes {
 	fn parse(input: syn::parse::ParseStream) -> Result<Self> {
-		let attrs = Attribute::parse_outer(input)?;
+		let lookahead = input.lookahead1();
+		Ok(if lookahead.peek(kw::consumes) {
+			Self {
+				keyword_consumes: Some(input.parse()?),
+				colon: input.parse()?,
+				consumes: input.parse()?,
+			}
+		} else {
+			Self { keyword_consumes: None, colon: None, consumes: input.parse()? }
+		})
+	}
+}
+
+/// Parses `(Foo, sends = [Bar, Baz])`
+/// including the `(` and `)`.
+#[derive(Debug, Clone)]
+pub(crate) struct SubSystemAttrItems {
+	/// The subsystem is in progress, only generate the `Wrapper` variant, but do not forward messages
+	/// and also not include the subsystem in the list of subsystems.
+	pub(crate) wip: bool,
+	/// If there are blocking components in the subsystem and hence it should be
+	/// spawned on a dedicated thread pool for such subssytems.
+	pub(crate) blocking: bool,
+	/// The message type being consumed by the subsystem.
+	pub(crate) consumes: Option<Consumes>,
+	pub(crate) sends: Option<Sends>,
+}
+
+impl Parse for SubSystemAttrItems {
+	fn parse(input: syn::parse::ParseStream) -> Result<Self> {
+		let span = input.span();
 
-		let input = input;
 		let content;
-		let _ = syn::parenthesized!(content in input);
+		let _paren_token = parenthesized!(content in input);
 
-		let mut items = Punctuated::new();
-		while let Ok(tag) = content.call(SubSysAttrItem::parse) {
-			items.push_value(tag);
-			items.push_punct(content.call(<Token![,]>::parse)?);
-		}
-
-		assert!(items.empty_or_trailing(), "Always followed by the message type to consume. qed");
-
-		let consumes = content.parse::<Path>()?;
+		let items = content.call(Punctuated::<SubSysAttrItem, Token![,]>::parse_terminated)?;
 
 		let mut unique = HashMap::<
 			std::mem::Discriminant<SubSysAttrItem>,
 			SubSysAttrItem,
 			RandomState,
 		>::default();
+
 		for item in items {
 			if let Some(first) = unique.insert(std::mem::discriminant(&item), item.clone()) {
-				let mut e = Error::new(
-					item.span(),
-					format!("Duplicate definition of subsystem attribute found"),
-				);
+				let mut e =
+					Error::new(item.span(), "Duplicate definition of subsystem attribute found");
 				e.combine(Error::new(first.span(), "previously defined here."));
 				return Err(e)
 			}
 		}
 
-		let no_dispatch = extract_variant!(unique, NoDispatch; default = false);
+		// A subsystem makes no sense if not one of them is provided
+		let sends = extract_variant!(unique, Sends take);
+		let consumes = extract_variant!(unique, Consumes take);
+		if sends.as_ref().map(|sends| sends.sends.is_empty()).unwrap_or(true) && consumes.is_none()
+		{
+			return Err(Error::new(
+				span,
+				"Must have at least one of `consumes: [..]` and `sends: [..]`.",
+			))
+		}
+
 		let blocking = extract_variant!(unique, Blocking; default = false);
 		let wip = extract_variant!(unique, Wip; default = false);
 
-		Ok(Self { attrs, no_dispatch, blocking, consumes, wip })
+		Ok(Self { blocking, wip, sends, consumes })
 	}
 }
 
@@ -192,7 +263,7 @@ pub(crate) struct BaggageField {
 #[derive(Clone, Debug)]
 pub(crate) struct OverseerInfo {
 	/// Where the support crate `::polkadot_overseer_gen` lives.
-	pub(crate) support_crate_name: TokenStream,
+	pub(crate) support_crate: Path,
 
 	/// Fields annotated with `#[subsystem(..)]`.
 	pub(crate) subsystems: Vec<SubSysField>,
@@ -216,11 +287,8 @@ pub(crate) struct OverseerInfo {
 	/// Incoming event type from the outer world, usually an external framework of some sort.
 	pub(crate) extern_event_ty: Path,
 
-	/// Incoming event type from an external entity, commonly from the network.
-	pub(crate) extern_network_ty: Option<Path>,
-
 	/// Type of messages that are sent to an external subsystem.
-	/// Merely here to be included during generation of `message_wrapper` type.
+	/// Merely here to be included during generation of `#message_wrapper` type.
 	pub(crate) outgoing_ty: Option<Path>,
 
 	/// Incoming event type from the outer world, commonly from the network.
@@ -228,8 +296,8 @@ pub(crate) struct OverseerInfo {
 }
 
 impl OverseerInfo {
-	pub(crate) fn support_crate_name(&self) -> &TokenStream {
-		&self.support_crate_name
+	pub(crate) fn support_crate_name(&self) -> &Path {
+		&self.support_crate
 	}
 
 	pub(crate) fn variant_names(&self) -> Vec<Ident> {
@@ -297,8 +365,11 @@ impl OverseerInfo {
 			.collect::<Vec<_>>()
 	}
 
-	pub(crate) fn consumes(&self) -> Vec<Path> {
-		self.subsystems.iter().map(|ssf| ssf.consumes.clone()).collect::<Vec<_>>()
+	pub(crate) fn any_message(&self) -> Vec<Path> {
+		self.subsystems
+			.iter()
+			.map(|ssf| ssf.message_to_consume.clone())
+			.collect::<Vec<_>>()
 	}
 
 	pub(crate) fn channel_names_without_wip(&self, suffix: &'static str) -> Vec<Ident> {
@@ -313,7 +384,7 @@ impl OverseerInfo {
 		self.subsystems
 			.iter()
 			.filter(|ssf| !ssf.wip)
-			.map(|ssf| ssf.consumes.clone())
+			.map(|ssf| ssf.message_to_consume.clone())
 			.collect::<Vec<_>>()
 	}
 }
@@ -341,7 +412,8 @@ impl OverseerGuts {
 		// for the builder pattern besides other places.
 		let mut unique_subsystem_idents = HashSet::<Ident>::new();
 		for Field { attrs, vis, ident, ty, .. } in fields.named.into_iter() {
-			let mut consumes =
+			// collect all subsystem annotations per field
+			let mut subsystem_attr =
 				attrs.iter().filter(|attr| attr.style == AttrStyle::Outer).filter_map(|attr| {
 					let span = attr.path.span();
 					attr.path.get_ident().filter(|ident| *ident == "subsystem").map(move |_ident| {
@@ -349,53 +421,75 @@ impl OverseerGuts {
 						(attr_tokens, span)
 					})
 				});
-			let ident =
-				ident.ok_or_else(|| Error::new(ty.span(), "Missing identifier for member. BUG"))?;
+			let ident = ident.ok_or_else(|| {
+				Error::new(
+					ty.span(),
+					"Missing identifier for field, only named fields are expceted.",
+				)
+			})?;
 
-			if let Some((attr_tokens, span)) = consumes.next() {
-				if let Some((_attr_tokens2, span2)) = consumes.next() {
+			// a `#[subsystem(..)]` annotation exists
+			if let Some((attr_tokens, span)) = subsystem_attr.next() {
+				if let Some((_attr_tokens2, span2)) = subsystem_attr.next() {
 					return Err({
 						let mut err = Error::new(span, "The first subsystem annotation is at");
 						err.combine(Error::new(span2, "but another here for the same field."));
 						err
 					})
 				}
-				let mut consumes_paths = Vec::with_capacity(attrs.len());
+
+				let span = attr_tokens.span();
+
 				let attr_tokens = attr_tokens.clone();
-				let variant: SubSystemTags = syn::parse2(attr_tokens.clone())?;
-				consumes_paths.push(variant.consumes);
+				let subsystem_attrs: SubSystemAttrItems = syn::parse2(attr_tokens.clone())?;
 
 				let field_ty = try_type_to_path(ty, span)?;
 				let generic = field_ty
 					.get_ident()
 					.ok_or_else(|| {
-						Error::new(field_ty.span(), "Must be an identifier, not a path.")
+						Error::new(
+							field_ty.span(),
+							"Must be an identifier, not a path. It will be used as a generic.",
+						)
 					})?
 					.clone();
+				// check for unique subsystem name, otherwise we'd create invalid code:
 				if let Some(previous) = unique_subsystem_idents.get(&generic) {
-					let mut e = Error::new(
-						generic.span(),
-						format!("Duplicate subsystem names `{}`", generic),
-					);
+					let mut e = Error::new(generic.span(), "Duplicate subsystem names");
 					e.combine(Error::new(previous.span(), "previously defined here."));
 					return Err(e)
 				}
 				unique_subsystem_idents.insert(generic.clone());
 
+				let SubSystemAttrItems { wip, blocking, consumes, sends, .. } = subsystem_attrs;
+
+				// messages to be sent
+				let sends = if let Some(sends) = sends {
+					Vec::from_iter(sends.sends.iter().cloned())
+				} else {
+					vec![]
+				};
+				// messages deemed for consumption
+				let consumes = if let Some(consumes) = consumes {
+					consumes.consumes
+				} else {
+					return Err(Error::new(span, "Must provide exactly one consuming message type"))
+				};
+
 				subsystems.push(SubSysField {
 					name: ident,
 					generic,
-					consumes: consumes_paths[0].clone(),
-					no_dispatch: variant.no_dispatch,
-					wip: variant.wip,
-					blocking: variant.blocking,
+					message_to_consume: consumes,
+					messages_to_send: sends,
+					wip,
+					blocking,
 				});
 			} else {
 				let field_ty = try_type_to_path(ty, ident.span())?;
 				let generic = field_ty
 					.get_ident()
 					.map(|ident| baggage_generics.contains(ident))
-					.unwrap_or_default();
+					.unwrap_or(false);
 				baggage.push(BaggageField { field_name: ident, generic, field_ty, vis });
 			}
 		}

polkadot/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs

@@ -0,0 +1,144 @@
+// Copyright 2022 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use super::kw;
+use proc_macro2::Span;
+use quote::{quote, ToTokens};
+use std::collections::{hash_map::RandomState, HashMap};
+use syn::{
+	parse::{Parse, ParseBuffer},
+	punctuated::Punctuated,
+	spanned::Spanned,
+	Error, Ident, Path, Result, Token,
+};
+
+#[derive(Clone, Debug)]
+enum SubsystemAttrItem {
+	/// Error type provided by the user.
+	Error { tag: kw::error, eq_token: Token![=], value: Path },
+	/// For which slot in the overseer this should be plugged.
+	///
+	/// The subsystem implementation can and should have a different name
+	/// from the declared parameter type in the overseer.
+	Subsystem { tag: Option<kw::subsystem>, eq_token: Option<Token![=]>, value: Ident },
+	/// The prefix to apply when a subsystem is implemented in a different file/crate
+	/// than the overseer itself.
+	///
+	/// Important for `#[subsystem(..)]` to reference the traits correctly.
+	TraitPrefix { tag: kw::prefix, eq_token: Token![=], value: Path },
+}
+
+impl ToTokens for SubsystemAttrItem {
+	fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) {
+		let ts = match self {
+			Self::TraitPrefix { tag, eq_token, value } => {
+				quote! { #tag #eq_token, #value }
+			},
+			Self::Error { tag, eq_token, value } => {
+				quote! { #tag #eq_token, #value }
+			},
+			Self::Subsystem { tag, eq_token, value } => {
+				quote! { #tag #eq_token, #value }
+			},
+		};
+		tokens.extend(ts.into_iter());
+	}
+}
+
+impl Parse for SubsystemAttrItem {
+	fn parse(input: &ParseBuffer) -> Result<Self> {
+		let lookahead = input.lookahead1();
+		if lookahead.peek(kw::error) {
+			Ok(SubsystemAttrItem::Error {
+				tag: input.parse::<kw::error>()?,
+				eq_token: input.parse()?,
+				value: input.parse()?,
+			})
+		} else if lookahead.peek(kw::prefix) {
+			Ok(SubsystemAttrItem::TraitPrefix {
+				tag: input.parse::<kw::prefix>()?,
+				eq_token: input.parse()?,
+				value: input.parse()?,
+			})
+		} else if lookahead.peek(kw::subsystem) {
+			Ok(SubsystemAttrItem::Subsystem {
+				tag: Some(input.parse::<kw::subsystem>()?),
+				eq_token: Some(input.parse()?),
+				value: input.parse()?,
+			})
+		} else {
+			Ok(SubsystemAttrItem::Subsystem { tag: None, eq_token: None, value: input.parse()? })
+		}
+	}
+}
+
+/// Attribute arguments `$args` in `#[subsystem( $args )]`.
+#[derive(Clone, Debug)]
+pub(crate) struct SubsystemAttrArgs {
+	span: Span,
+	pub(crate) error_path: Option<Path>,
+	pub(crate) subsystem_ident: Ident,
+	pub(crate) trait_prefix_path: Option<Path>,
+}
+
+impl Spanned for SubsystemAttrArgs {
+	fn span(&self) -> Span {
+		self.span.clone()
+	}
+}
+
+macro_rules! extract_variant {
+	($unique:expr, $variant:ident ; default = $fallback:expr) => {
+		extract_variant!($unique, $variant).unwrap_or_else(|| $fallback)
+	};
+	($unique:expr, $variant:ident ; err = $err:expr) => {
+		extract_variant!($unique, $variant).ok_or_else(|| Error::new(Span::call_site(), $err))
+	};
+	($unique:expr, $variant:ident) => {
+		$unique.values().find_map(|item| match item {
+			SubsystemAttrItem::$variant { value, .. } => Some(value.clone()),
+			_ => None,
+		})
+	};
+}
+
+impl Parse for SubsystemAttrArgs {
+	fn parse(input: &ParseBuffer) -> Result<Self> {
+		let span = input.span();
+		let items: Punctuated<SubsystemAttrItem, Token![,]> =
+			input.parse_terminated(SubsystemAttrItem::parse)?;
+
+		let mut unique = HashMap::<
+			std::mem::Discriminant<SubsystemAttrItem>,
+			SubsystemAttrItem,
+			RandomState,
+		>::default();
+		for item in items {
+			if let Some(first) = unique.insert(std::mem::discriminant(&item), item.clone()) {
+				let mut e = Error::new(
+					item.span(),
+					format!("Duplicate definition of subsystem generation type found"),
+				);
+				e.combine(Error::new(first.span(), "previously defined here."));
+				return Err(e)
+			}
+		}
+		let error_path = extract_variant!(unique, Error);
+		let subsystem_ident = extract_variant!(unique, Subsystem; err = "Must annotate the identical overseer error type via `subsystem=..` or plainly as `Subsystem` as specified in the overseer declaration.")?;
+		let trait_prefix_path = extract_variant!(unique, TraitPrefix);
+		Ok(SubsystemAttrArgs { span, error_path, subsystem_ident, trait_prefix_path })
+	}
+}

polkadot/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs

@@ -0,0 +1,295 @@
+// Copyright 2022 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use super::*;
+use crate::{SubSysAttrItem, SubSystemAttrItems};
+use assert_matches::assert_matches;
+use quote::quote;
+use syn::parse_quote;
+
+mod attr {
+	use super::*;
+
+	#[test]
+	fn attr_full_works() {
+		let attr: OverseerAttrArgs = parse_quote! {
+			gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222,
+			error=OverseerError,
+		};
+		assert_matches!(attr, OverseerAttrArgs {
+			message_channel_capacity,
+			signal_channel_capacity,
+			..
+		} => {
+			assert_eq!(message_channel_capacity, 222);
+			assert_eq!(signal_channel_capacity, 111);
+		});
+	}
+
+	#[test]
+	fn attr_partial_works() {
+		let attr: OverseerAttrArgs = parse_quote! {
+			gen=AllMessage, event=::some::why::ExternEvent, signal=::foo::SigSigSig,
+			error=OverseerError,
+		};
+		assert_matches!(attr, OverseerAttrArgs {
+			message_channel_capacity: _,
+			signal_channel_capacity: _,
+			..
+		} => {
+		});
+	}
+}
+
+mod strukt {
+
+	use super::*;
+
+	#[test]
+	fn parse_subsystem_attr_item_works_00_wip() {
+		assert_matches!(
+		syn::parse2::<SubSysAttrItem>(quote! {
+			wip
+		}), Ok(SubSysAttrItem::Wip(_)) => {
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attr_item_works_02_sends() {
+		assert_matches!(
+		syn::parse2::<SubSysAttrItem>(quote! {
+			sends: [A, B, C]
+		}), Ok(SubSysAttrItem::Sends(sends)) => {
+			assert_eq!(sends.sends.len(), 3);
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attr_item_works_03_sends() {
+		assert_matches!(
+		syn::parse2::<SubSysAttrItem>(quote! {
+			sends: [A]
+		}), Ok(SubSysAttrItem::Sends(sends)) => {
+			assert_eq!(sends.sends.len(), 1);
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attr_item_works_04_sends() {
+		assert_matches!(
+		syn::parse2::<SubSysAttrItem>(quote! {
+			sends: [A,]
+		}), Ok(SubSysAttrItem::Sends(sends)) => {
+			assert_eq!(sends.sends.len(), 1);
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attr_item_works_05_sends() {
+		assert_matches!(
+		syn::parse2::<SubSysAttrItem>(quote! {
+			sends: []
+		}), Ok(SubSysAttrItem::Sends(sends)) => {
+			assert_eq!(sends.sends.len(), 0);
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attr_item_works_06_consumes() {
+		assert_matches!(
+		syn::parse2::<SubSysAttrItem>(quote! {
+			consumes: Foo
+		}), Ok(SubSysAttrItem::Consumes(_consumes)) => {
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attr_item_works_07_consumes() {
+		assert_matches!(
+		syn::parse2::<SubSysAttrItem>(quote! {
+			Foo
+		}), Ok(SubSysAttrItem::Consumes(_consumes)) => {
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attributes_works_00() {
+		syn::parse2::<SubSystemAttrItems>(quote! {
+			(wip, blocking, consumes: Foo, sends: [])
+		})
+		.unwrap();
+	}
+
+	#[test]
+	fn parse_subsystem_attributes_works_01() {
+		assert_matches!(
+		syn::parse2::<SubSystemAttrItems>(quote! {
+			(blocking, Foo, sends: [])
+		}), Ok(_) => {
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attributes_works_02() {
+		assert_matches!(
+		syn::parse2::<SubSystemAttrItems>(quote! {
+			(consumes: Foo, sends: [Bar])
+		}), Ok(_) => {
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attributes_works_03() {
+		assert_matches!(
+		syn::parse2::<SubSystemAttrItems>(quote! {
+			(blocking, consumes: Foo, sends: [Bar])
+		}), Ok(_) => {
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attributes_works_04() {
+		assert_matches!(
+		syn::parse2::<SubSystemAttrItems>(quote! {
+			(wip, consumes: Foo, sends: [Bar])
+		}), Ok(_) => {
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attributes_works_05() {
+		assert_matches!(
+		syn::parse2::<SubSystemAttrItems>(quote! {
+			(consumes: Foo)
+		}), Ok(_) => {
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attributes_works_06() {
+		assert_matches!(
+		syn::parse2::<SubSystemAttrItems>(quote! {
+			(sends: [Foo], consumes: Bar)
+		}), Ok(_) => {
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attributes_works_07_duplicate_send() {
+		assert_matches!(
+		syn::parse2::<SubSystemAttrItems>(quote! {
+			(sends: [Foo], Bar, Y)
+		}), Err(e) => {
+			dbg!(e)
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attributes_works_08() {
+		assert_matches!(
+		syn::parse2::<SubSystemAttrItems>(quote! {
+			(sends: [Foo], consumes: Bar)
+		}), Ok(_) => {
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attributes_works_09_neither_consumes_nor_sends() {
+		assert_matches!(
+		syn::parse2::<SubSystemAttrItems>(quote! {
+			(sends: [])
+		}), Err(e) => {
+			// must either consume smth or sends smth, neither is NOK
+			dbg!(e)
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attributes_works_10_empty_with_braces() {
+		assert_matches!(
+		syn::parse2::<SubSystemAttrItems>(quote! {
+			()
+		}), Err(e) => {
+			dbg!(e)
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attributes_works_11_empty() {
+		assert_matches!(
+		syn::parse2::<SubSystemAttrItems>(quote! {
+
+		}), Err(e) => {
+			dbg!(e)
+		});
+	}
+
+	#[test]
+	fn parse_subsystem_attributes_works_12_duplicate_consumes_different_fmt() {
+		assert_matches!(
+		syn::parse2::<SubSystemAttrItems>(quote! {
+			(Foo, consumes = Foo)
+		}), Err(e) => {
+			dbg!(e)
+		});
+	}
+
+	#[test]
+	fn struct_parse_baggage() {
+		let item: OverseerGuts = parse_quote! {
+			pub struct Ooooh<X = Pffffffft> where X: Secrit {
+				#[subsystem(consumes: Foo, sends: [])]
+				sub0: FooSubsystem,
+
+				metrics: Metrics,
+			}
+		};
+		let _ = dbg!(item);
+	}
+
+	#[test]
+	fn struct_parse_full() {
+		let item: OverseerGuts = parse_quote! {
+			pub struct Ooooh<X = Pffffffft> where X: Secrit {
+				#[subsystem(consumes: Foo, sends: [])]
+				sub0: FooSubsystem,
+
+				#[subsystem(blocking, consumes: Bar, sends: [])]
+				yyy: BaersBuyBilliardBalls,
+
+				#[subsystem(blocking, consumes: Twain, sends: [])]
+				fff: Beeeeep,
+
+				#[subsystem(consumes: Rope)]
+				mc: MountainCave,
+
+				metrics: Metrics,
+			}
+		};
+		let _ = dbg!(item);
+	}
+
+	#[test]
+	fn struct_parse_basic() {
+		let item: OverseerGuts = parse_quote! {
+			pub struct Ooooh {
+				#[subsystem(consumes: Foo, sends: [])]
+				sub0: FooSubsystem,
+			}
+		};
+		let _ = dbg!(item);
+	}
+}

polkadot/node/overseer/overseer-gen/proc-macro/src/subsystem.rs

@@ -0,0 +1,310 @@
+// Copyright 2022 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Generates the bounds for a particular subsystem `Context` and associate `type Sender`.
+//!
+//!
+//! ## Implement `trait Subsystem<Context, Error>` via `subsystem`
+//!
+//! ```ignore
+//! # use polkadot_overseer_gen_proc_macro::subsystem;
+//! # mod somewhere {
+//! # use polkadot_overseer_gen_proc_macro::overlord;
+//! # pub use polkadot_overseer_gen::*;
+//! #
+//! # #[derive(Debug, thiserror::Error)]
+//! # #[error("Yikes!")]
+//! # pub struct Yikes;
+//! # impl From<OverseerError> for Yikes {
+//! #   fn from(_: OverseerError) -> Yikes { Yikes }
+//! # }
+//! # impl From<mpsc::SendError> for Yikes {
+//! #   fn from(_: mpsc::SendError) -> Yikes { Yikes }
+//! # }
+//! #
+//! # #[derive(Debug)]
+//! # pub struct Eve;
+//! #
+//! # #[derive(Debug, Clone)]
+//! # pub struct Sig;
+//! #
+//! # #[derive(Debug, Clone, Copy)]
+//! # pub struct A;
+//! # #[derive(Debug, Clone, Copy)]
+//! # pub struct B;
+//! #
+//! # #[overlord(signal=Sig, gen=AllOfThem, event=Eve, error=Yikes)]
+//! # pub struct Wonderland {
+//! # 	#[subsystem(A, sends: [B])]
+//! # 	foo: Foo,
+//! # 	#[subsystem(B, sends: [A])]
+//! # 	bar: Bar,
+//! # }
+//! # }
+//! # use somewhere::{Yikes, SpawnedSubsystem};
+//! #
+//! # struct FooSubsystem;
+//! #
+//! #[subsystem(Foo, error = Yikes, prefix = somewhere)]
+//! impl<Context> FooSubsystem {
+//!    fn start(self, context: Context) -> SpawnedSubsystem<Yikes> {
+//!	       // ..
+//!        # let _ = context;
+//!        # unimplemented!()
+//!    }
+//! }
+//! ```
+//!
+//! expands to
+//!
+//! ```ignore
+//! # use polkadot_overseer_gen_proc_macro::subsystem;
+//! # mod somewhere {
+//! # use polkadot_overseer_gen_proc_macro::overlord;
+//! # pub use polkadot_overseer_gen::*;
+//! #
+//! # #[derive(Debug, thiserror::Error)]
+//! # #[error("Yikes!")]
+//! # pub struct Yikes;
+//! # impl From<OverseerError> for Yikes {
+//! #   fn from(_: OverseerError) -> Yikes { Yikes }
+//! # }
+//! # impl From<mpsc::SendError> for Yikes {
+//! #   fn from(_: mpsc::SendError) -> Yikes { Yikes }
+//! # }
+//! #
+//! # #[derive(Debug)]
+//! # pub struct Eve;
+//! #
+//! # #[derive(Debug, Clone)]
+//! # pub struct Sig;
+//! #
+//! # #[derive(Debug, Clone, Copy)]
+//! # pub struct A;
+//! # #[derive(Debug, Clone, Copy)]
+//! # pub struct B;
+//! #
+//! # #[overlord(signal=Sig, gen=AllOfThem, event=Eve, error=Yikes)]
+//! # pub struct Wonderland {
+//! # 	#[subsystem(A, sends: [B])]
+//! # 	foo: Foo,
+//! # 	#[subsystem(B, sends: [A])]
+//! # 	bar: Bar,
+//! # }
+//! # }
+//! # use somewhere::{Yikes, SpawnedSubsystem};
+//! # use polkadot_overseer_gen as support_crate;
+//! #
+//! # struct FooSubsystem;
+//! #
+//! impl<Context> support_crate::Subsystem<Context, Yikes> for FooSubsystem
+//! where
+//! 	Context: somewhere::FooContextTrait,
+//!     Context: support_crate::SubsystemContext,
+//!		<Context as somewhere::FooContextTrait>::Sender: somewhere::FooSenderTrait,
+//!		<Context as support_crate::SubsystemContext>::Sender: somewhere::FooSenderTrait,
+//! {
+//!       fn start(self, context: Context) -> SpawnedSubsystem<Yikes> {
+//!        // ..
+//!        # let _ = context;
+//!        # unimplemented!()
+//!       }
+//! }
+//! ```
+//!
+//! where `support_crate` is either equivalent to `somewhere` or derived from the cargo manifest.
+//!
+//!
+//! ## Add additional trait bounds for a generic `Context` via `contextbounds`
+//!
+//! ### To an `ImplItem`
+//!
+//! ```ignore
+//! #[contextbounds(Foo, prefix = somewhere)]
+//! impl<Context> X {
+//! ..
+//! }
+//! ```
+//!
+//! expands to
+//!
+//! ```ignore
+//! impl<Context> X
+//! where
+//! 	Context: somewhere::FooSubsystemTrait,
+//!     Context: support_crate::SubsystemContext,
+//!		<Context as somewhere::FooContextTrait>::Sender: somewhere::FooSenderTrait,
+//!		<Context as support_crate::SubsystemContext>::Sender: somewhere::FooSenderTrait,
+//! {
+//! }
+//! ```
+//!
+//! ### To a free standing `Fn` (not a method, that's covered by the above)
+//!
+//! ```ignore
+//! #[contextbounds(Foo, prefix = somewhere)]
+//! fn do_smth<Context>(context: &mut Context) {
+//! ..
+//! }
+//! ```
+//!
+//! expands to
+//!
+//! ```ignore
+//! fn do_smth<Context>(context: &mut Context)
+//! where
+//! 	Context: somewhere::FooSubsystemTrait,
+//!     Context: support_crate::SubsystemContext,
+//!		<Context as somewhere::FooContextTrait>::Sender: somewhere::FooSenderTrait,
+//!		<Context as support_crate::SubsystemContext>::Sender: somewhere::FooSenderTrait,
+//! {
+//! }
+//! ```
+use proc_macro2::TokenStream;
+use quote::{format_ident, ToTokens};
+use syn::{parse2, parse_quote, punctuated::Punctuated, Result};
+
+use super::{parse::*, *};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum MakeSubsystem {
+	/// Implements `trait Subsystem` and apply the trait bounds to the `Context` generic.
+	///
+	/// Relevant to `impl Item` only.
+	ImplSubsystemTrait,
+	/// Only apply the trait bounds to the context.
+	AddContextTraitBounds,
+}
+
+pub(crate) fn impl_subsystem_context_trait_bounds(
+	attr: TokenStream,
+	orig: TokenStream,
+	make_subsystem: MakeSubsystem,
+) -> Result<proc_macro2::TokenStream> {
+	let args = parse2::<SubsystemAttrArgs>(attr.clone())?;
+	let span = args.span();
+	let SubsystemAttrArgs { error_path, subsystem_ident, trait_prefix_path, .. } = args;
+
+	let mut item = parse2::<syn::Item>(orig)?;
+
+	// always prefer the direct usage, if it's not there, let's see if there is
+	// a `prefix=*` provided. Either is ok.
+
+	// Technically this is two different things:
+	// The place where the `#[overlord]` is annotated is where all `trait *SenderTrait` and
+	// `trait *ContextTrait` types exist.
+	// The other usage is the true support crate `polkadot-overseer-gen`, where the static ones
+	// are declared.
+	// Right now, if the `support_crate` is not included, it falls back silently to the `trait_prefix_path`.
+	let support_crate = support_crate()
+		.or_else(|_e| {
+			trait_prefix_path.clone().ok_or_else(|| {
+				syn::Error::new(attr.span(), "Couldn't find `polkadot-overseer-gen` in manifest, but also missing a `prefix=` to help trait bound resolution")
+			})
+		})?;
+
+	let trait_prefix_path = trait_prefix_path.unwrap_or_else(|| parse_quote! { self });
+	if trait_prefix_path.segments.trailing_punct() {
+		return Err(syn::Error::new(trait_prefix_path.span(), "Must not end with `::`"))
+	}
+
+	let subsystem_ctx_trait = format_ident!("{}ContextTrait", subsystem_ident);
+	let subsystem_sender_trait = format_ident!("{}SenderTrait", subsystem_ident);
+
+	let extra_where_predicates: Punctuated<syn::WherePredicate, syn::Token![,]> = parse_quote! {
+		Context: #trait_prefix_path::#subsystem_ctx_trait,
+		Context: #support_crate::SubsystemContext,
+		<Context as #trait_prefix_path::#subsystem_ctx_trait>::Sender: #trait_prefix_path::#subsystem_sender_trait,
+		<Context as #support_crate::SubsystemContext>::Sender: #trait_prefix_path::#subsystem_sender_trait,
+	};
+
+	let apply_ctx_bound_if_present = move |generics: &mut syn::Generics| -> bool {
+		if generics
+			.params
+			.iter()
+			.find(|generic| match generic {
+				syn::GenericParam::Type(ty) if ty.ident == "Context" => true,
+				_ => false,
+			})
+			.is_some()
+		{
+			let where_clause = generics.make_where_clause();
+			where_clause.predicates.extend(extra_where_predicates.clone());
+			true
+		} else {
+			false
+		}
+	};
+
+	match item {
+		syn::Item::Impl(ref mut struktured_impl) => {
+			if make_subsystem == MakeSubsystem::ImplSubsystemTrait {
+				let error_path = error_path.ok_or_else(|| {
+					syn::Error::new(
+						span,
+						"Must annotate the identical overseer error type via `error=..`.",
+					)
+				})?;
+				// Only replace the subsystem trait if it's desired.
+				struktured_impl.trait_.replace((
+					None,
+					parse_quote! {
+						#support_crate::Subsystem<Context, #error_path>
+					},
+					syn::token::For::default(),
+				));
+			}
+
+			apply_ctx_bound_if_present(&mut struktured_impl.generics);
+			for item in struktured_impl.items.iter_mut() {
+				match item {
+					syn::ImplItem::Method(method) => {
+						apply_ctx_bound_if_present(&mut method.sig.generics);
+					},
+					_others => {
+						// don't error, just nop
+					},
+				}
+			}
+		},
+		syn::Item::Fn(ref mut struktured_fn) => {
+			if make_subsystem == MakeSubsystem::ImplSubsystemTrait {
+				return Err(syn::Error::new(struktured_fn.span(), "Cannot make a free function a subsystem, did you mean to apply `contextbound` instead?"))
+			}
+			apply_ctx_bound_if_present(&mut struktured_fn.sig.generics);
+		},
+		other =>
+			return Err(syn::Error::new(
+				other.span(),
+				"Macro can only be annotated on functions or struct implementations",
+			)),
+	};
+
+	Ok(item.to_token_stream())
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn is_path() {
+		let _p: Path = parse_quote! { self };
+		let _p: Path = parse_quote! { crate };
+		let _p: Path = parse_quote! { ::foo };
+		let _p: Path = parse_quote! { bar };
+	}
+}

polkadot/node/overseer/overseer-gen/proc-macro/src/tests.rs

@@ -32,13 +32,13 @@ fn print() {
 
 	let item = quote! {
 		pub struct Ooooh<X = Pffffffft> where X: Secrit {
-			#[subsystem(no_dispatch, Foo)]
+			#[subsystem(Foo)]
 			sub0: FooSubsystem,
 
 			#[subsystem(blocking, Bar)]
 			yyy: BaersBuyBilliardBalls,
 
-			#[subsystem(no_dispatch, blocking, Twain)]
+			#[subsystem(blocking, Twain)]
 			fff: Beeeeep,
 
 			#[subsystem(Rope)]
@@ -57,13 +57,13 @@ fn print() {
 fn struct_parse_full() {
 	let item: OverseerGuts = parse_quote! {
 		pub struct Ooooh<X = Pffffffft> where X: Secrit {
-			#[subsystem(no_dispatch, Foo)]
+			#[subsystem(Foo)]
 			sub0: FooSubsystem,
 
 			#[subsystem(blocking, Bar)]
 			yyy: BaersBuyBilliardBalls,
 
-			#[subsystem(no_dispatch, blocking, Twain)]
+			#[subsystem(blocking, Twain)]
 			fff: Beeeeep,
 
 			#[subsystem(Rope)]
@@ -88,11 +88,11 @@ fn struct_parse_basic() {
 
 #[test]
 fn attr_full() {
-	let attr: AttrArgs = parse_quote! {
+	let attr: OverseerAttrArgs = parse_quote! {
 		gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222,
 		error=OverseerError,
 	};
-	assert_matches!(attr, AttrArgs {
+	assert_matches!(attr, OverseerAttrArgs {
 		message_channel_capacity,
 		signal_channel_capacity,
 		..
@@ -104,11 +104,11 @@ fn attr_full() {
 
 #[test]
 fn attr_partial() {
-	let attr: AttrArgs = parse_quote! {
+	let attr: OverseerAttrArgs = parse_quote! {
 		gen=AllMessage, event=::some::why::ExternEvent, signal=::foo::SigSigSig,
 		error=OverseerError,
 	};
-	assert_matches!(attr, AttrArgs {
+	assert_matches!(attr, OverseerAttrArgs {
 		message_channel_capacity: _,
 		signal_channel_capacity: _,
 		..

polkadot/node/overseer/overseer-gen/src/lib.rs

@@ -60,12 +60,13 @@
 #![deny(missing_docs)]
 #![deny(unused_crate_dependencies)]
 
-pub use polkadot_overseer_gen_proc_macro::overlord;
+pub use polkadot_overseer_gen_proc_macro::{contextbounds, overlord, subsystem};
 
 #[doc(hidden)]
 pub use gum;
 #[doc(hidden)]
 pub use metered;
+
 #[doc(hidden)]
 pub use polkadot_node_primitives::SpawnNamed;
 
@@ -101,7 +102,7 @@ use std::fmt;
 #[cfg(test)]
 mod tests;
 
-/// A type of messages that are sent from [`Subsystem`] to [`Overseer`].
+/// A type of messages that are sent from a [`Subsystem`] to the declared overseer.
 ///
 /// Used to launch jobs.
 pub enum ToOverseer {
@@ -312,7 +313,7 @@ pub struct SubsystemMeterReadouts {
 ///
 /// [`Subsystem`]: trait.Subsystem.html
 ///
-/// `M` here is the inner message type, and _not_ the generated `enum AllMessages`.
+/// `M` here is the inner message type, and _not_ the generated `enum AllMessages` or `#message_wrapper` type.
 pub struct SubsystemInstance<Message, Signal> {
 	/// Send sink for `Signal`s to be sent to a subsystem.
 	pub tx_signal: crate::metered::MeteredSender<Signal>,
@@ -362,20 +363,23 @@ pub trait SubsystemContext: Send + 'static {
 	/// The message type of this context. Subsystems launched with this context will expect
 	/// to receive messages of this type. Commonly uses the wrapping `enum` commonly called
 	/// `AllMessages`.
-	type Message: std::fmt::Debug + Send + 'static;
+	type Message: ::std::fmt::Debug + Send + 'static;
 	/// And the same for signals.
-	type Signal: std::fmt::Debug + Send + 'static;
-	/// The overarching all messages `enum`.
-	/// In some cases can be identical to `Self::Message`.
-	type AllMessages: From<Self::Message> + Send + 'static;
+	type Signal: ::std::fmt::Debug + Send + 'static;
+	/// The overarching messages `enum` for this particular subsystem.
+	type OutgoingMessages: ::std::fmt::Debug + Send + 'static;
+
+	// The overarching messages `enum` for this particular subsystem.
+	// type AllMessages: From<Self::OutgoingMessages> + From<Self::Message> + std::fmt::Debug + Send + 'static;
+
 	/// The sender type as provided by `sender()` and underlying.
-	type Sender: SubsystemSender<Self::AllMessages> + Send + 'static;
+	type Sender: Clone + Send + 'static + SubsystemSender<Self::OutgoingMessages>;
 	/// The error type.
 	type Error: ::std::error::Error + ::std::convert::From<OverseerError> + Sync + Send + 'static;
 
 	/// Try to asynchronously receive a message.
 	///
-	/// This has to be used with caution, if you loop over this without
+	/// Has to be used with caution, if you loop over this without
 	/// using `pending!()` macro you will end up with a busy loop!
 	async fn try_recv(&mut self) -> Result<Option<FromOverseer<Self::Message, Self::Signal>>, ()>;
 
@@ -397,34 +401,37 @@ pub trait SubsystemContext: Send + 'static {
 	) -> Result<(), Self::Error>;
 
 	/// Send a direct message to some other `Subsystem`, routed based on message type.
-	async fn send_message<X>(&mut self, msg: X)
+	// #[deprecated(note = "Use `self.sender().send_message(msg) instead, avoid passing around the full context.")]
+	async fn send_message<T>(&mut self, msg: T)
 	where
-		Self::AllMessages: From<X>,
-		X: Send,
+		Self::OutgoingMessages: From<T> + Send,
+		T: Send,
 	{
-		self.sender().send_message(<Self::AllMessages>::from(msg)).await
+		self.sender().send_message(<Self::OutgoingMessages>::from(msg)).await
 	}
 
 	/// Send multiple direct messages to other `Subsystem`s, routed based on message type.
-	async fn send_messages<X, T>(&mut self, msgs: T)
+	// #[deprecated(note = "Use `self.sender().send_message(msg) instead, avoid passing around the full context.")]
+	async fn send_messages<T, I>(&mut self, msgs: I)
 	where
-		T: IntoIterator<Item = X> + Send,
-		T::IntoIter: Send,
-		Self::AllMessages: From<X>,
-		X: Send,
+		Self::OutgoingMessages: From<T> + Send,
+		I: IntoIterator<Item = T> + Send,
+		I::IntoIter: Send,
+		T: Send,
 	{
 		self.sender()
-			.send_messages(msgs.into_iter().map(|x| <Self::AllMessages>::from(x)))
+			.send_messages(msgs.into_iter().map(<Self::OutgoingMessages>::from))
 			.await
 	}
 
 	/// Send a message using the unbounded connection.
+	// #[deprecated(note = "Use `self.sender().send_unbounded_message(msg) instead, avoid passing around the full context.")]
 	fn send_unbounded_message<X>(&mut self, msg: X)
 	where
-		Self::AllMessages: From<X>,
+		Self::OutgoingMessages: From<X> + Send,
 		X: Send,
 	{
-		self.sender().send_unbounded_message(Self::AllMessages::from(msg))
+		self.sender().send_unbounded_message(<Self::OutgoingMessages>::from(msg))
 	}
 
 	/// Obtain the sender.
@@ -450,22 +457,25 @@ where
 
 /// Sender end of a channel to interface with a subsystem.
 #[async_trait::async_trait]
-pub trait SubsystemSender<Message>: Send + Clone + 'static {
+pub trait SubsystemSender<OutgoingMessage>: Clone + Send + 'static
+where
+	OutgoingMessage: Send,
+{
 	/// Send a direct message to some other `Subsystem`, routed based on message type.
-	async fn send_message(&mut self, msg: Message);
+	async fn send_message(&mut self, msg: OutgoingMessage);
 
 	/// Send multiple direct messages to other `Subsystem`s, routed based on message type.
-	async fn send_messages<T>(&mut self, msgs: T)
+	async fn send_messages<I>(&mut self, msgs: I)
 	where
-		T: IntoIterator<Item = Message> + Send,
-		T::IntoIter: Send;
+		I: IntoIterator<Item = OutgoingMessage> + Send,
+		I::IntoIter: Send;
 
 	/// Send a message onto the unbounded queue of some other `Subsystem`, routed based on message
 	/// type.
 	///
 	/// This function should be used only when there is some other bounding factor on the messages
 	/// sent with it. Otherwise, it risks a memory leak.
-	fn send_unbounded_message(&mut self, msg: Message);
+	fn send_unbounded_message(&mut self, msg: OutgoingMessage);
 }
 
 /// A future that wraps another future with a `Delay` allowing for time-limited futures.

polkadot/node/overseer/overseer-gen/tests/ui/err-04-missing-error.rs

@@ -15,7 +15,7 @@ struct MsgStrukt(u8);
 
 #[overlord(signal=SigSigSig, event=Event, gen=AllMessages)]
 struct Overseer {
-	#[subsystem(no_dispatch, MsgStrukt)]
+	#[subsystem(MsgStrukt)]
 	sub0: AwesomeSubSys,
 
 	i_like_pie: f64,

polkadot/node/overseer/overseer-gen/tests/ui/err-05-missing-field.rs

@@ -21,7 +21,7 @@ pub struct MsgStrukt(u8);
 
 #[overlord(signal=SigSigSig, error=OverseerError, event=Event, gen=AllMessages)]
 struct Overseer {
-	#[subsystem(no_dispatch, MsgStrukt)]
+	#[subsystem(MsgStrukt)]
 	sub0: AwesomeSubSys,
 	i_like_pie: f64,
 }
@@ -58,4 +58,4 @@ fn main() {
 		.spawner(DummySpawner)
 		.build()
 		.unwrap();
-}
+}

polkadot/node/overseer/overseer-gen/tests/ui/err-06-missing-subsystem.rs

@@ -21,7 +21,7 @@ pub struct MsgStrukt(u8);
 
 #[overlord(signal=SigSigSig, error=OverseerError, event=Event, gen=AllMessages)]
 struct Overseer {
-	#[subsystem(no_dispatch, MsgStrukt)]
+	#[subsystem(MsgStrukt)]
 	sub0: AwesomeSubSys,
 	i_like_pie: f64,
 }

polkadot/node/overseer/overseer-gen/tests/ui/err-07-missing-spawner.rs

@@ -21,7 +21,7 @@ pub struct MsgStrukt(u8);
 
 #[overlord(signal=SigSigSig, error=OverseerError, event=Event, gen=AllMessages)]
 struct Overseer {
-	#[subsystem(no_dispatch, MsgStrukt)]
+	#[subsystem(MsgStrukt)]
 	sub0: AwesomeSubSys,
 	i_like_pie: f64,
 }

polkadot/node/overseer/overseer-gen/tests/ui/err-08-duplicate-subsystem.rs

@@ -21,7 +21,7 @@ pub struct MsgStrukt(u8);
 
 #[overlord(signal=SigSigSig, error=OverseerError, event=Event, gen=AllMessages)]
 struct Overseer {
-	#[subsystem(no_dispatch, MsgStrukt)]
+	#[subsystem(MsgStrukt)]
 	sub0: AwesomeSubSys,
 	i_like_pie: f64,
 }

polkadot/node/overseer/overseer-gen/tests/ui/err-09-uninit_generic_baggage.rs

@@ -21,7 +21,7 @@ pub struct MsgStrukt(u8);
 
 #[overlord(signal=SigSigSig, error=OverseerError, event=Event, gen=AllMessages)]
 struct Overseer<T> {
-	#[subsystem(no_dispatch, MsgStrukt)]
+	#[subsystem(MsgStrukt)]
 	sub0: AwesomeSubSys,
 	i_like_pie: T,
 }