Service factory refactor (#3382)

* Move Service::new to a macro

* Move function calls to macros

* Extract offchain_workers and start_rpc in separate function

In follow-up commits, we want to be able to directly call maintain_transaction_pool, offchain_workers, and start_rpc, without having to implement the Components trait.
This commit is a preliminary step: we extract the code to freestanding functions.

* Introduce an AbstractService trait

* Introduce NewService as an implementation detail of Service

* Implement traits on NewService instead

Instead of implementing AbstractService, Future, and Executor on Service, we implement them on NewService instead.

The implementations of AbstractService, Future, and Executor on Service still exist, but they just wrap to the respective implementations for NewService.

* Move components creation back to macro invocation

Instead of having multiple $build_ parameters passed to the macro, let's group them all into one.

This change is necessary for the follow-up commits, because we are going to call new_impl! only after all the components have already been built.

* Add a $block parameter to new_impl

This makes it possible to be explicit as what the generic parameter of the NewServiceis, without relying on type inference.

* Introduce the ServiceBuilder struct

Introduces a new builder-like ServiceBuilder struct that creates a NewService.

* Macro-ify import_blocks, export_blocks and revert_chain

Similar to the introduction of new_impl!, we extract the actual code into a macro, letting us get rid of the Components and Factory traits

* Add export_blocks, import_blocks and revert_chain methods on ServiceBuilder

Can be used as a replacement for the chain_ops::* methods

* Add run_with_builder

Instead of just run, adds run_with_builder to ParseAndPrepareExport/Import/Revert. This lets you run these operations with a ServiceBuilder instead of a ServiceFactory.

* Transition node and node-template to ServiceBuilder

* Transition transaction-factory to the new service factory

This is technically a breaking change, but the transaction-factory crate is only ever used from within substrate-node, which this commit updates as well.

* Remove old service factory

* Adjust the AbstractService trait to be more usable

We slightly change the trait bounds in order to make all the methods usable.

* Make substrate-service-test compile

* Fix the node-cli tests

* Remove the old API

* Remove the components module

* Fix indentation on chain_ops

* Line widths

* Fix bad line widths commit

* Line widths again 🤦

* Fix the sync test

* Apply suggestions from code review

Co-Authored-By: Gavin Wood <i@gavwood.com>

* Address some concerns

* Remove TelemetryOnConnect

* Remove informant::start

* Update jsonrpc

* Rename factory to builder

* Line widths 😩

substrate/test-utils/transaction-factory/src/complex_mode.rs

@@ -41,29 +41,30 @@
 use std::sync::Arc;
 
 use log::info;
+use client::Client;
 use client::block_builder::api::BlockBuilder;
 use client::runtime_api::ConstructRuntimeApi;
+use primitives::{Blake2Hasher, Hasher};
 use sr_primitives::generic::BlockId;
 use sr_primitives::traits::{Block as BlockT, ProvideRuntimeApi, One, Zero};
-use substrate_service::{
-	FactoryBlock, FullClient, ServiceFactory, ComponentClient, FullComponents
-};
 
 use crate::{RuntimeAdapter, create_block};
 
-pub fn next<F, RA>(
+pub fn next<RA, Backend, Exec, Block, RtApi>(
 	factory_state: &mut RA,
-	client: &Arc<ComponentClient<FullComponents<F>>>,
+	client: &Arc<Client<Backend, Exec, Block, RtApi>>,
 	version: u32,
 	genesis_hash: <RA::Block as BlockT>::Hash,
 	prior_block_hash: <RA::Block as BlockT>::Hash,
-	prior_block_id: BlockId<F::Block>,
-) -> Option<<F as ServiceFactory>::Block>
+	prior_block_id: BlockId<Block>,
+) -> Option<Block>
 where
-	F: ServiceFactory,
-	F::RuntimeApi: ConstructRuntimeApi<FactoryBlock<F>, FullClient<F>>,
-	FullClient<F>: ProvideRuntimeApi,
-	<FullClient<F> as ProvideRuntimeApi>::Api: BlockBuilder<FactoryBlock<F>>,
+	Block: BlockT<Hash = <Blake2Hasher as Hasher>::Out>,
+	Exec: client::CallExecutor<Block, Blake2Hasher> + Send + Sync + Clone,
+	Backend: client::backend::Backend<Block, Blake2Hasher> + Send,
+	Client<Backend, Exec, Block, RtApi>: ProvideRuntimeApi,
+	<Client<Backend, Exec, Block, RtApi> as ProvideRuntimeApi>::Api: BlockBuilder<Block>,
+	RtApi: ConstructRuntimeApi<Block, Client<Backend, Exec, Block, RtApi>> + Send + Sync,
 	RA: RuntimeAdapter,
 {
 	let total = factory_state.start_number() + factory_state.num() * factory_state.rounds();
@@ -102,7 +103,7 @@ where
 	let inherents = client.runtime_api().inherent_extrinsics(&prior_block_id, inherents)
 		.expect("Failed to create inherent extrinsics");
 
-	let block = create_block::<F, RA>(&client, transfer, inherents);
+	let block = create_block::<RA, _, _, _, _>(&client, transfer, inherents);
 	info!(
 		"Created block {} with hash {}. Transferring {} from {} to {}.",
 		factory_state.block_no() + RA::Number::one(),

substrate/test-utils/transaction-factory/src/lib.rs

@@ -26,22 +26,19 @@ use std::fmt::Display;
 
 use log::info;
 
-use client::block_builder::api::BlockBuilder;
-use client::runtime_api::ConstructRuntimeApi;
+use client::{Client, block_builder::api::BlockBuilder, runtime_api::ConstructRuntimeApi};
 use consensus_common::{
 	BlockOrigin, BlockImportParams, InherentData, ForkChoiceStrategy,
 	SelectChain
 };
 use consensus_common::block_import::BlockImport;
 use codec::{Decode, Encode};
+use primitives::{Blake2Hasher, Hasher};
 use sr_primitives::generic::BlockId;
 use sr_primitives::traits::{
 	Block as BlockT, Header as HeaderT, ProvideRuntimeApi, SimpleArithmetic,
 	One, Zero,
 };
-use substrate_service::{
-	FactoryBlock, FactoryFullConfiguration, FullClient, new_client,
-	ServiceFactory, ComponentClient, FullComponents};
 pub use crate::modes::Mode;
 
 pub mod modes;
@@ -95,15 +92,19 @@ pub trait RuntimeAdapter {
 
 /// Manufactures transactions. The exact amount depends on
 /// `mode`, `num` and `rounds`.
-pub fn factory<F, RA>(
+pub fn factory<RA, Backend, Exec, Block, RtApi, Sc>(
 	mut factory_state: RA,
-	mut config: FactoryFullConfiguration<F>,
+	client: &Arc<Client<Backend, Exec, Block, RtApi>>,
+	select_chain: &Sc,
 ) -> cli::error::Result<()>
 where
-	F: ServiceFactory,
-	F::RuntimeApi: ConstructRuntimeApi<FactoryBlock<F>, FullClient<F>>,
-	FullClient<F>: ProvideRuntimeApi,
-	<FullClient<F> as ProvideRuntimeApi>::Api: BlockBuilder<FactoryBlock<F>>,
+	Block: BlockT<Hash = <Blake2Hasher as Hasher>::Out>,
+	Exec: client::CallExecutor<Block, Blake2Hasher> + Send + Sync + Clone,
+	Backend: client::backend::Backend<Block, Blake2Hasher> + Send,
+	Client<Backend, Exec, Block, RtApi>: ProvideRuntimeApi,
+	<Client<Backend, Exec, Block, RtApi> as ProvideRuntimeApi>::Api: BlockBuilder<Block>,
+	RtApi: ConstructRuntimeApi<Block, Client<Backend, Exec, Block, RtApi>> + Send + Sync,
+	Sc: SelectChain<Block>,
 	RA: RuntimeAdapter,
 	<<RA as RuntimeAdapter>::Block as BlockT>::Hash: From<primitives::H256>,
 {
@@ -112,20 +113,16 @@ where
 		return Err(cli::error::Error::Input(msg));
 	}
 
-	let client = new_client::<F>(&config)?;
-
-	let select_chain = F::build_select_chain(&mut config, client.clone())?;
-
-	let best_header: Result<<F::Block as BlockT>::Header, cli::error::Error> =
+	let best_header: Result<<Block as BlockT>::Header, cli::error::Error> =
 		select_chain.best_chain().map_err(|e| format!("{:?}", e).into());
 	let mut best_hash = best_header?.hash();
-	let best_block_id = BlockId::<F::Block>::hash(best_hash);
+	let best_block_id = BlockId::<Block>::hash(best_hash);
 	let version = client.runtime_version_at(&best_block_id)?.spec_version;
 	let genesis_hash = client.block_hash(Zero::zero())?
 		.expect("Genesis block always exists; qed").into();
 
 	while let Some(block) = match factory_state.mode() {
-		Mode::MasterToNToM => complex_mode::next::<F, RA>(
+		Mode::MasterToNToM => complex_mode::next::<RA, _, _, _, _>(
 			&mut factory_state,
 			&client,
 			version,
@@ -133,7 +130,7 @@ where
 			best_hash.into(),
 			best_block_id,
 		),
-		_ => simple_modes::next::<F, RA>(
+		_ => simple_modes::next::<RA, _, _, _, _>(
 			&mut factory_state,
 			&client,
 			version,
@@ -143,7 +140,7 @@ where
 		),
 	} {
 		best_hash = block.header().hash();
-		import_block::<F>(&client, block);
+		import_block(&client, block);
 
 		info!("Imported block at {}", factory_state.block_no());
 	}
@@ -152,16 +149,18 @@ where
 }
 
 /// Create a baked block from a transfer extrinsic and timestamp inherent.
-pub fn create_block<F, RA>(
-	client: &Arc<ComponentClient<FullComponents<F>>>,
+pub fn create_block<RA, Backend, Exec, Block, RtApi>(
+	client: &Arc<Client<Backend, Exec, Block, RtApi>>,
 	transfer: <RA::Block as BlockT>::Extrinsic,
-	inherent_extrinsics: Vec<<F::Block as BlockT>::Extrinsic>,
-) -> <F as ServiceFactory>::Block
+	inherent_extrinsics: Vec<<Block as BlockT>::Extrinsic>,
+) -> Block
 where
-	F: ServiceFactory,
-	FullClient<F>: ProvideRuntimeApi,
-	F::RuntimeApi: ConstructRuntimeApi<FactoryBlock<F>, FullClient<F>>,
-	<FullClient<F> as ProvideRuntimeApi>::Api: BlockBuilder<FactoryBlock<F>>,
+	Block: BlockT<Hash = <Blake2Hasher as Hasher>::Out>,
+	Exec: client::CallExecutor<Block, Blake2Hasher> + Send + Sync + Clone,
+	Backend: client::backend::Backend<Block, Blake2Hasher> + Send,
+	Client<Backend, Exec, Block, RtApi>: ProvideRuntimeApi,
+	RtApi: ConstructRuntimeApi<Block, Client<Backend, Exec, Block, RtApi>> + Send + Sync,
+	<Client<Backend, Exec, Block, RtApi> as ProvideRuntimeApi>::Api: BlockBuilder<Block>,
 	RA: RuntimeAdapter,
 {
 	let mut block = client.new_block(Default::default()).expect("Failed to create new block");
@@ -177,10 +176,13 @@ where
 	block.bake().expect("Failed to bake block")
 }
 
-fn import_block<F>(
-	client: &Arc<ComponentClient<FullComponents<F>>>,
-	block: <F as ServiceFactory>::Block
-) -> () where F: ServiceFactory
+fn import_block<Backend, Exec, Block, RtApi>(
+	client: &Arc<Client<Backend, Exec, Block, RtApi>>,
+	block: Block
+) -> () where 
+	Block: BlockT<Hash = <Blake2Hasher as Hasher>::Out>,
+	Exec: client::CallExecutor<Block, Blake2Hasher> + Send + Sync + Clone,
+	Backend: client::backend::Backend<Block, Blake2Hasher> + Send,
 {
 	let import = BlockImportParams {
 		origin: BlockOrigin::File,

substrate/test-utils/transaction-factory/src/simple_modes.rs

@@ -36,29 +36,30 @@
 use std::sync::Arc;
 
 use log::info;
+use client::Client;
 use client::block_builder::api::BlockBuilder;
 use client::runtime_api::ConstructRuntimeApi;
+use primitives::{Blake2Hasher, Hasher};
 use sr_primitives::traits::{Block as BlockT, ProvideRuntimeApi, One};
 use sr_primitives::generic::BlockId;
-use substrate_service::{
-	FactoryBlock, FullClient, ServiceFactory, ComponentClient, FullComponents
-};
 
 use crate::{Mode, RuntimeAdapter, create_block};
 
-pub fn next<F, RA>(
+pub fn next<RA, Backend, Exec, Block, RtApi>(
 	factory_state: &mut RA,
-	client: &Arc<ComponentClient<FullComponents<F>>>,
+	client: &Arc<Client<Backend, Exec, Block, RtApi>>,
 	version: u32,
 	genesis_hash: <RA::Block as BlockT>::Hash,
 	prior_block_hash: <RA::Block as BlockT>::Hash,
-	prior_block_id: BlockId<F::Block>,
-) -> Option<<F as ServiceFactory>::Block>
+	prior_block_id: BlockId<Block>,
+) -> Option<Block>
 where
-	F: ServiceFactory,
-	F::RuntimeApi: ConstructRuntimeApi<FactoryBlock<F>, FullClient<F>>,
-	FullClient<F>: ProvideRuntimeApi,
-	<FullClient<F> as ProvideRuntimeApi>::Api: BlockBuilder<FactoryBlock<F>>,
+	Block: BlockT<Hash = <Blake2Hasher as Hasher>::Out>,
+	Exec: client::CallExecutor<Block, Blake2Hasher> + Send + Sync + Clone,
+	Backend: client::backend::Backend<Block, Blake2Hasher> + Send,
+	Client<Backend, Exec, Block, RtApi>: ProvideRuntimeApi,
+	<Client<Backend, Exec, Block, RtApi> as ProvideRuntimeApi>::Api: BlockBuilder<Block>,
+	RtApi: ConstructRuntimeApi<Block, Client<Backend, Exec, Block, RtApi>> + Send + Sync,
 	RA: RuntimeAdapter,
 {
 	if factory_state.block_no() >= factory_state.num() {
@@ -93,7 +94,7 @@ where
 	let inherents = client.runtime_api().inherent_extrinsics(&prior_block_id, inherents)
 		.expect("Failed to create inherent extrinsics");
 
-	let block = create_block::<F, RA>(&client, transfer, inherents);
+	let block = create_block::<RA, _, _, _, _>(&client, transfer, inherents);
 
 	factory_state.set_block_no(factory_state.block_no() + RA::Number::one());