overseer becomes orchestra (#5542)

* rename overseer-gen to orchestra

Also drop `gum` and use `tracing`.

* make orchestra compile as standalone

* introduce Spawner trait to split from sp_core

Finalizes the independence of orchestra from polkadot-overseer

* slip of the pen

* other fixins

* remove unused import

* Update node/overseer/orchestra/proc-macro/src/impl_builder.rs

Co-authored-by: Vsevolod Stakhov <vsevolod.stakhov@parity.io>

* Update node/overseer/orchestra/proc-macro/src/impl_builder.rs

Co-authored-by: Vsevolod Stakhov <vsevolod.stakhov@parity.io>

* orchestra everywhere

* leaky data

* Bump scale-info from 2.1.1 to 2.1.2 (#5552)

Bumps [scale-info](https://github.com/paritytech/scale-info) from 2.1.1 to 2.1.2.
- [Release notes](https://github.com/paritytech/scale-info/releases)
- [Changelog](https://github.com/paritytech/scale-info/blob/master/CHANGELOG.md)
- [Commits](https://github.com/paritytech/scale-info/compare/v2.1.1...v2.1.2)

---
updated-dependencies:
- dependency-name: scale-info
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* Add missing markdown code block delimiter (#5555)

* bitfield-signing: remove util::jobs usage  (#5523)

* Switch to pooling copy-on-write instantiation strategy for WASM (companion for Substrate#11232) (#5337)

* Switch to pooling copy-on-write instantiation strategy for WASM

* Fix compilation of `polkadot-test-service`

* Update comments

* Move `max_memory_size` to `Semantics`

* Rename `WasmInstantiationStrategy` to `WasmtimeInstantiationStrategy`

* Update a safety comment

* update lockfile for {"substrate"}

Co-authored-by: parity-processbot <>

* Fix build

Co-authored-by: Vsevolod Stakhov <vsevolod.stakhov@parity.io>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Malte Kliemann <mail@maltekliemann.com>
Co-authored-by: Chris Sosnin <48099298+slumber@users.noreply.github.com>
Co-authored-by: Koute <koute@users.noreply.github.com>

polkadot/node/malus/src/interceptor.rs

@@ -21,7 +21,7 @@
 //! messages on the overseer level.
 
 use polkadot_node_subsystem::*;
-pub use polkadot_node_subsystem::{messages, messages::*, overseer, FromOverseer};
+pub use polkadot_node_subsystem::{messages, messages::*, overseer, FromOrchestra};
 use std::{future::Future, pin::Pin};
 
 /// Filter incoming and outgoing messages.
@@ -42,8 +42,8 @@ where
 	fn intercept_incoming(
 		&self,
 		_sender: &mut Sender,
-		msg: FromOverseer<Self::Message>,
-	) -> Option<FromOverseer<Self::Message>> {
+		msg: FromOrchestra<Self::Message>,
+	) -> Option<FromOrchestra<Self::Message>> {
 		Some(msg)
 	}
 
@@ -174,7 +174,7 @@ where
 	type OutgoingMessages = <<Context as overseer::SubsystemContext>::Message as overseer::AssociateOutgoing>::OutgoingMessages;
 	type Signal = OverseerSignal;
 
-	async fn try_recv(&mut self) -> Result<Option<FromOverseer<Self::Message>>, ()> {
+	async fn try_recv(&mut self) -> Result<Option<FromOrchestra<Self::Message>>, ()> {
 		loop {
 			match self.inner.try_recv().await? {
 				None => return Ok(None),
@@ -188,7 +188,7 @@ where
 		}
 	}
 
-	async fn recv(&mut self) -> SubsystemResult<FromOverseer<Self::Message>> {
+	async fn recv(&mut self) -> SubsystemResult<FromOrchestra<Self::Message>> {
 		loop {
 			let msg = self.inner.recv().await?;
 			if let Some(msg) = self.message_filter.intercept_incoming(self.inner.sender(), msg) {
@@ -243,10 +243,6 @@ where
 			<Context as overseer::SubsystemContext>::Sender,
 			Message = <Context as overseer::SubsystemContext>::Message,
 		>,
-	// <Context as overseer::SubsystemContext>::Sender:
-	// 	overseer::SubsystemSender<
-	// 		<Interceptor as MessageInterceptor<<Context as overseer::SubsystemContext>::Sender>>::Message,
-	// 	>,
 	<Context as overseer::SubsystemContext>::Message:
 		overseer::AssociateOutgoing,
 	<Context as overseer::SubsystemContext>::Sender:

polkadot/node/malus/src/shared.rs

@@ -15,7 +15,7 @@
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
 use futures::prelude::*;
-use polkadot_node_primitives::SpawnNamed;
+use sp_core::traits::SpawnNamed;
 
 pub const MALUS: &str = "MALUS";
 

polkadot/node/malus/src/tests.rs

@@ -37,10 +37,10 @@ where
 	fn intercept_incoming(
 		&self,
 		_sender: &mut Sender,
-		msg: FromOverseer<Self::Message>,
-	) -> Option<FromOverseer<Self::Message>> {
+		msg: FromOrchestra<Self::Message>,
+	) -> Option<FromOrchestra<Self::Message>> {
 		match msg {
-			FromOverseer::Communication { msg: _msg } => None,
+			FromOrchestra::Communication { msg: _msg } => None,
 			// to conclude the test cleanly
 			sig => Some(sig),
 		}
@@ -60,18 +60,20 @@ where
 }
 
 async fn overseer_send<T: Into<AllMessages>>(overseer: &mut TestSubsystemContextHandle<T>, msg: T) {
-	overseer.send(FromOverseer::Communication { msg }).await;
+	overseer.send(FromOrchestra::Communication { msg }).await;
 }
 
+use sp_core::testing::TaskExecutor;
+
 fn launch_harness<F, M, Sub, G>(test_gen: G)
 where
 	F: Future<Output = TestSubsystemContextHandle<M>> + Send,
 	M: AssociateOutgoing + std::fmt::Debug + Send + 'static,
 	// <M as AssociateOutgoing>::OutgoingMessages: From<M>,
-	Sub: Subsystem<TestSubsystemContext<M, sp_core::testing::TaskExecutor>, SubsystemError>,
+	Sub: Subsystem<TestSubsystemContext<M, SpawnGlue<TaskExecutor>>, SubsystemError>,
 	G: Fn(TestSubsystemContextHandle<M>) -> (F, Sub),
 {
-	let pool = sp_core::testing::TaskExecutor::new();
+	let pool = TaskExecutor::new();
 	let (context, overseer) = make_subsystem_context(pool);
 
 	let (test_fut, subsystem) = test_gen(overseer);
@@ -84,7 +86,7 @@ where
 	futures::executor::block_on(futures::future::join(
 		async move {
 			let mut overseer = test_fut.await;
-			overseer.send(FromOverseer::Signal(OverseerSignal::Conclude)).await;
+			overseer.send(FromOrchestra::Signal(OverseerSignal::Conclude)).await;
 		},
 		subsystem,
 	))

polkadot/node/malus/src/variants/back_garbage_candidate.rs

@@ -23,9 +23,11 @@ use polkadot_cli::{
 	service::{
 		AuthorityDiscoveryApi, AuxStore, BabeApi, Block, Error, HeaderBackend, Overseer,
 		OverseerConnector, OverseerGen, OverseerGenArgs, OverseerHandle, ParachainHost,
-		ProvideRuntimeApi, SpawnNamed,
+		ProvideRuntimeApi,
 	},
 };
+use polkadot_node_subsystem::SpawnGlue;
+use sp_core::traits::SpawnNamed;
 
 use crate::{
 	interceptor::*,
@@ -43,7 +45,7 @@ impl OverseerGen for BackGarbageCandidate {
 		&self,
 		connector: OverseerConnector,
 		args: OverseerGenArgs<'a, Spawner, RuntimeClient>,
-	) -> Result<(Overseer<Spawner, Arc<RuntimeClient>>, OverseerHandle), Error>
+	) -> Result<(Overseer<SpawnGlue<Spawner>, Arc<RuntimeClient>>, OverseerHandle), Error>
 	where
 		RuntimeClient: 'static + ProvideRuntimeApi<Block> + HeaderBackend<Block> + AuxStore,
 		RuntimeClient::Api: ParachainHost<Block> + BabeApi<Block> + AuthorityDiscoveryApi<Block>,
@@ -53,7 +55,7 @@ impl OverseerGen for BackGarbageCandidate {
 		let validation_filter = ReplaceValidationResult::new(
 			FakeCandidateValidation::BackingAndApprovalValid,
 			FakeCandidateValidationError::InvalidOutputs,
-			spawner.clone(),
+			SpawnGlue(spawner),
 		);
 
 		prepared_overseer_builder(args)?

polkadot/node/malus/src/variants/common.rs

@@ -23,14 +23,15 @@ use crate::{
 
 use polkadot_node_core_candidate_validation::find_validation_data;
 use polkadot_node_primitives::{InvalidCandidate, ValidationResult};
-use polkadot_node_subsystem::messages::{CandidateValidationMessage, ValidationFailed};
+use polkadot_node_subsystem::{
+	messages::{CandidateValidationMessage, ValidationFailed},
+	overseer,
+};
 
 use polkadot_primitives::v2::{
 	CandidateCommitments, CandidateDescriptor, CandidateReceipt, PersistedValidationData,
 };
 
-use polkadot_cli::service::SpawnNamed;
-
 use futures::channel::oneshot;
 
 #[derive(clap::ArgEnum, Clone, Copy, Debug, PartialEq)]
@@ -113,7 +114,7 @@ pub struct ReplaceValidationResult<Spawner> {
 
 impl<Spawner> ReplaceValidationResult<Spawner>
 where
-	Spawner: SpawnNamed,
+	Spawner: overseer::gen::Spawner,
 {
 	pub fn new(
 		fake_validation: FakeCandidateValidation,
@@ -197,7 +198,7 @@ fn create_validation_response(
 impl<Sender, Spawner> MessageInterceptor<Sender> for ReplaceValidationResult<Spawner>
 where
 	Sender: overseer::CandidateValidationSenderTrait + Clone + Send + 'static,
-	Spawner: SpawnNamed + Clone + 'static,
+	Spawner: overseer::gen::Spawner + Clone + 'static,
 {
 	type Message = CandidateValidationMessage;
 
@@ -205,10 +206,10 @@ where
 	fn intercept_incoming(
 		&self,
 		subsystem_sender: &mut Sender,
-		msg: FromOverseer<Self::Message>,
-	) -> Option<FromOverseer<Self::Message>> {
+		msg: FromOrchestra<Self::Message>,
+	) -> Option<FromOrchestra<Self::Message>> {
 		match msg {
-			FromOverseer::Communication {
+			FromOrchestra::Communication {
 				msg:
 					CandidateValidationMessage::ValidateFromExhaustive(
 						validation_data,
@@ -224,7 +225,7 @@ where
 					FakeCandidateValidation::BackingAndApprovalValid => {
 						// Behave normally if the `PoV` is not known to be malicious.
 						if pov.block_data.0.as_slice() != MALICIOUS_POV {
-							return Some(FromOverseer::Communication {
+							return Some(FromOrchestra::Communication {
 								msg: CandidateValidationMessage::ValidateFromExhaustive(
 									validation_data,
 									validation_code,
@@ -257,7 +258,7 @@ where
 						sender.send(Ok(validation_result)).unwrap();
 						None
 					},
-					_ => Some(FromOverseer::Communication {
+					_ => Some(FromOrchestra::Communication {
 						msg: CandidateValidationMessage::ValidateFromExhaustive(
 							validation_data,
 							validation_code,
@@ -269,7 +270,7 @@ where
 					}),
 				}
 			},
-			FromOverseer::Communication {
+			FromOrchestra::Communication {
 				msg:
 					CandidateValidationMessage::ValidateFromChainState(
 						candidate_receipt,
@@ -283,7 +284,7 @@ where
 					FakeCandidateValidation::BackingAndApprovalValid => {
 						// Behave normally if the `PoV` is not known to be malicious.
 						if pov.block_data.0.as_slice() != MALICIOUS_POV {
-							return Some(FromOverseer::Communication {
+							return Some(FromOrchestra::Communication {
 								msg: CandidateValidationMessage::ValidateFromChainState(
 									candidate_receipt,
 									pov,
@@ -314,7 +315,7 @@ where
 						response_sender.send(Ok(validation_result)).unwrap();
 						None
 					},
-					_ => Some(FromOverseer::Communication {
+					_ => Some(FromOrchestra::Communication {
 						msg: CandidateValidationMessage::ValidateFromChainState(
 							candidate_receipt,
 							pov,

polkadot/node/malus/src/variants/dispute_valid_candidates.rs

@@ -27,10 +27,12 @@ use polkadot_cli::{
 	service::{
 		AuthorityDiscoveryApi, AuxStore, BabeApi, Block, Error, HeaderBackend, Overseer,
 		OverseerConnector, OverseerGen, OverseerGenArgs, OverseerHandle, ParachainHost,
-		ProvideRuntimeApi, SpawnNamed,
+		ProvideRuntimeApi,
 	},
 	RunCmd,
 };
+use polkadot_node_subsystem::SpawnGlue;
+use sp_core::traits::SpawnNamed;
 
 // Filter wrapping related types.
 use super::common::{FakeCandidateValidation, FakeCandidateValidationError};
@@ -73,7 +75,7 @@ impl OverseerGen for DisputeValidCandidates {
 		&self,
 		connector: OverseerConnector,
 		args: OverseerGenArgs<'a, Spawner, RuntimeClient>,
-	) -> Result<(Overseer<Spawner, Arc<RuntimeClient>>, OverseerHandle), Error>
+	) -> Result<(Overseer<SpawnGlue<Spawner>, Arc<RuntimeClient>>, OverseerHandle), Error>
 	where
 		RuntimeClient: 'static + ProvideRuntimeApi<Block> + HeaderBackend<Block> + AuxStore,
 		RuntimeClient::Api: ParachainHost<Block> + BabeApi<Block> + AuthorityDiscoveryApi<Block>,
@@ -83,7 +85,7 @@ impl OverseerGen for DisputeValidCandidates {
 		let validation_filter = ReplaceValidationResult::new(
 			self.opts.fake_validation,
 			self.opts.fake_validation_error,
-			spawner.clone(),
+			SpawnGlue(spawner.clone()),
 		);
 
 		prepared_overseer_builder(args)?

polkadot/node/malus/src/variants/suggest_garbage_candidate.rs

@@ -27,7 +27,7 @@ use polkadot_cli::{
 	service::{
 		AuthorityDiscoveryApi, AuxStore, BabeApi, Block, Error, HeaderBackend, Overseer,
 		OverseerConnector, OverseerGen, OverseerGenArgs, OverseerHandle, ParachainHost,
-		ProvideRuntimeApi, SpawnNamed,
+		ProvideRuntimeApi,
 	},
 };
 use polkadot_node_core_candidate_validation::find_validation_data;
@@ -35,6 +35,7 @@ use polkadot_node_primitives::{AvailableData, BlockData, PoV};
 use polkadot_primitives::v2::{CandidateDescriptor, CandidateHash};
 
 use polkadot_node_subsystem_util::request_validators;
+use sp_core::traits::SpawnNamed;
 
 // Filter wrapping related types.
 use crate::{
@@ -48,7 +49,10 @@ use crate::{
 
 // Import extra types relevant to the particular
 // subsystem.
-use polkadot_node_subsystem::messages::{CandidateBackingMessage, CollatorProtocolMessage};
+use polkadot_node_subsystem::{
+	messages::{CandidateBackingMessage, CollatorProtocolMessage},
+	SpawnGlue,
+};
 use polkadot_primitives::v2::CandidateReceipt;
 
 use std::{
@@ -72,7 +76,7 @@ struct NoteCandidate<Spawner> {
 impl<Sender, Spawner> MessageInterceptor<Sender> for NoteCandidate<Spawner>
 where
 	Sender: overseer::CandidateBackingSenderTrait + Clone + Send + 'static,
-	Spawner: SpawnNamed + Clone + 'static,
+	Spawner: overseer::gen::Spawner + Clone + 'static,
 {
 	type Message = CandidateBackingMessage;
 
@@ -80,10 +84,10 @@ where
 	fn intercept_incoming(
 		&self,
 		subsystem_sender: &mut Sender,
-		msg: FromOverseer<Self::Message>,
-	) -> Option<FromOverseer<Self::Message>> {
+		msg: FromOrchestra<Self::Message>,
+	) -> Option<FromOrchestra<Self::Message>> {
 		match msg {
-			FromOverseer::Communication {
+			FromOrchestra::Communication {
 				msg: CandidateBackingMessage::Second(relay_parent, candidate, _pov),
 			} => {
 				gum::debug!(
@@ -204,14 +208,14 @@ where
 					.map
 					.insert(malicious_candidate_hash, candidate.hash());
 
-				let message = FromOverseer::Communication {
+				let message = FromOrchestra::Communication {
 					msg: CandidateBackingMessage::Second(relay_parent, malicious_candidate, pov),
 				};
 
 				Some(message)
 			},
-			FromOverseer::Communication { msg } => Some(FromOverseer::Communication { msg }),
-			FromOverseer::Signal(signal) => Some(FromOverseer::Signal(signal)),
+			FromOrchestra::Communication { msg } => Some(FromOrchestra::Communication { msg }),
+			FromOrchestra::Signal(signal) => Some(FromOrchestra::Signal(signal)),
 		}
 	}
 
@@ -245,7 +249,7 @@ impl OverseerGen for BackGarbageCandidateWrapper {
 		&self,
 		connector: OverseerConnector,
 		args: OverseerGenArgs<'a, Spawner, RuntimeClient>,
-	) -> Result<(Overseer<Spawner, Arc<RuntimeClient>>, OverseerHandle), Error>
+	) -> Result<(Overseer<SpawnGlue<Spawner>, Arc<RuntimeClient>>, OverseerHandle), Error>
 	where
 		RuntimeClient: 'static + ProvideRuntimeApi<Block> + HeaderBackend<Block> + AuxStore,
 		RuntimeClient::Api: ParachainHost<Block> + BabeApi<Block> + AuthorityDiscoveryApi<Block>,
@@ -254,12 +258,12 @@ impl OverseerGen for BackGarbageCandidateWrapper {
 		let inner = Inner { map: std::collections::HashMap::new() };
 		let inner_mut = Arc::new(Mutex::new(inner));
 		let note_candidate =
-			NoteCandidate { inner: inner_mut.clone(), spawner: args.spawner.clone() };
+			NoteCandidate { inner: inner_mut.clone(), spawner: SpawnGlue(args.spawner.clone()) };
 
 		let validation_filter = ReplaceValidationResult::new(
 			FakeCandidateValidation::BackingAndApprovalValid,
 			FakeCandidateValidationError::InvalidOutputs,
-			args.spawner.clone(),
+			SpawnGlue(args.spawner.clone()),
 		);
 
 		prepared_overseer_builder(args)?