establish new node folder for overseer, messages, and subsystems (#1200)

* establish new `node` folder for overseer, messages, and subsystems

* extract message types from overseer crate

* remove doc links

polkadot/node/core/README.md

@@ -0,0 +1 @@
+Stub - This folder will hold core subsystem implementations, each with their own crate.

polkadot/node/messages/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "polkadot-node-messages"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+description = "Message types used by Subsystems"
+
+[dependencies]
+polkadot-primitives = { path = "../../primitives" }

polkadot/node/messages/src/lib.rs

@@ -0,0 +1,72 @@
+// Copyright 2017-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Message types for the overseer and subsystems.
+//!
+//! These messages are intended to define the protocol by which different subsystems communicate with each
+//! other and signals that they receive from an overseer to coordinate their work.
+//! This is intended for use with the `polkadot-overseer` crate.
+//!
+//! Subsystems' APIs are defined separately from their implementation, leading to easier mocking.
+
+use polkadot_primitives::Hash;
+
+/// Signals sent by an overseer to a subsystem.
+#[derive(PartialEq, Clone, Debug)]
+pub enum OverseerSignal {
+	/// `Subsystem` should start working on block-based work, given by the relay-chain block hash.
+	StartWork(Hash),
+	/// `Subsystem` should stop working on block-based work specified by the relay-chain block hash.
+	StopWork(Hash),
+	/// Conclude the work of the `Overseer` and all `Subsystem`s.
+	Conclude,
+}
+
+/// A message type used by the Validation Subsystem.
+#[derive(Debug)]
+pub enum ValidationSubsystemMessage {
+	ValidityAttestation,
+}
+
+/// A message type used by the CandidateBacking Subsystem.
+#[derive(Debug)]
+pub enum CandidateBackingSubsystemMessage {
+	RegisterBackingWatcher,
+	Second,
+}
+
+/// A message type tying together all message types that are used across Subsystems.
+#[derive(Debug)]
+pub enum AllMessages {
+	Validation(ValidationSubsystemMessage),
+	CandidateBacking(CandidateBackingSubsystemMessage),
+}
+
+/// A message type that a subsystem receives from an overseer.
+/// It wraps signals from an overseer and messages that are circulating
+/// between subsystems.
+///
+/// It is generic over over the message type `M` that a particular `Subsystem` may use.
+#[derive(Debug)]
+pub enum FromOverseer<M: std::fmt::Debug> {
+	/// Signal from the `Overseer`.
+	Signal(OverseerSignal),
+
+	/// Some other `Subsystem`'s message.
+	Communication {
+		msg: M,
+	},
+}

polkadot/node/network/README.md

@@ -0,0 +1 @@
+Stub - This folder will hold networking subsystem implementations, each with their own crate.

polkadot/node/overseer/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "polkadot-overseer"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+futures = "0.3.5"
+log = "0.4.8"
+futures-timer = "3.0.2"
+streamunordered = "0.5.1"
+polkadot-primitives = { path = "../../primitives" }
+client = { package = "sc-client-api", git = "https://github.com/paritytech/substrate", branch = "master" }
+messages = { package = "polkadot-node-messages", path = "../messages" }
+
+[dev-dependencies]
+futures = { version = "0.3.5", features = ["thread-pool"] }
+futures-timer = "3.0.2"
+femme = "2.0.1"
+log = "0.4.8"
+kv-log-macro = "1.0.6"

polkadot/node/overseer/examples/minimal-example.rs

@@ -0,0 +1,136 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Shows a basic usage of the `Overseer`:
+//!   * Spawning subsystems and subsystem child jobs
+//!   * Establishing message passing
+
+use std::time::Duration;
+use futures::{
+	pending, pin_mut, executor, select, stream,
+	FutureExt, StreamExt,
+};
+use futures_timer::Delay;
+use kv_log_macro as log;
+
+use polkadot_overseer::{Overseer, Subsystem, SubsystemContext, SpawnedSubsystem};
+
+use messages::{
+	AllMessages, CandidateBackingSubsystemMessage, FromOverseer, ValidationSubsystemMessage
+};
+
+struct Subsystem1;
+
+impl Subsystem1 {
+	async fn run(mut ctx: SubsystemContext<CandidateBackingSubsystemMessage>)  {
+		loop {
+			match ctx.try_recv().await {
+				Ok(Some(msg)) => {
+					if let FromOverseer::Communication { msg } = msg {
+						log::info!("msg {:?}", msg);
+					}
+					continue;
+				}
+				Ok(None) => (),
+				Err(_) => {
+					log::info!("exiting");
+					return;
+				}
+			}
+
+			Delay::new(Duration::from_secs(1)).await;
+			ctx.send_msg(AllMessages::Validation(
+				ValidationSubsystemMessage::ValidityAttestation
+			)).await.unwrap();
+		}
+	}
+}
+
+impl Subsystem<CandidateBackingSubsystemMessage> for Subsystem1 {
+	fn start(&mut self, ctx: SubsystemContext<CandidateBackingSubsystemMessage>) -> SpawnedSubsystem {
+		SpawnedSubsystem(Box::pin(async move {
+			Self::run(ctx).await;
+		}))
+	}
+}
+
+struct Subsystem2;
+
+impl Subsystem2 {
+	async fn run(mut ctx: SubsystemContext<ValidationSubsystemMessage>)  {
+		ctx.spawn(Box::pin(async {
+			loop {
+				log::info!("Job tick");
+				Delay::new(Duration::from_secs(1)).await;
+			}
+		})).await.unwrap();
+
+		loop {
+			match ctx.try_recv().await {
+				Ok(Some(msg)) => {
+					log::info!("Subsystem2 received message {:?}", msg);
+					continue;
+				}
+				Ok(None) => { pending!(); }
+				Err(_) => {
+					log::info!("exiting");
+					return;
+				},
+			}
+		}
+	}
+}
+
+impl Subsystem<ValidationSubsystemMessage> for Subsystem2 {
+	fn start(&mut self, ctx: SubsystemContext<ValidationSubsystemMessage>) -> SpawnedSubsystem {
+		SpawnedSubsystem(Box::pin(async move {
+			Self::run(ctx).await;
+		}))
+	}
+}
+
+fn main() {
+	femme::with_level(femme::LevelFilter::Trace);
+	let spawner = executor::ThreadPool::new().unwrap();
+
+	futures::executor::block_on(async {
+		let timer_stream = stream::repeat(()).then(|_| async {
+			Delay::new(Duration::from_secs(1)).await;
+		});
+
+		let (overseer, _handler) = Overseer::new(
+			vec![],
+			Box::new(Subsystem2),
+			Box::new(Subsystem1),
+			spawner,
+		).unwrap();
+		let overseer_fut = overseer.run().fuse();
+		let timer_stream = timer_stream;
+
+		pin_mut!(timer_stream);
+		pin_mut!(overseer_fut);
+
+		loop {
+			select! {
+				_ = overseer_fut => break,
+				_ = timer_stream.next() => {
+					log::info!("tick");
+				}
+				complete => break,
+			}
+		}
+	});
+}