Integration tests (#805)

* Started substrate tests

* Sync test

* Test updates

* Improved tests

* Use on-chain block delay

* Parallel test execution

* Otimized tests

* Logging

* Fixed racing test

* Fixed compilation

* Fixed timestamp test

* Removed rlp dependency

* Minor fixes

* Fixed tests

* Removed best_block_id and resolved fdlimit issue

* Whitespace

* Use keyring

* Style

* Added API execution setting

* Removed stale import

substrate/core/service/src/chain_spec.rs

@@ -30,6 +30,16 @@ enum GenesisSource<G> {
 	Factory(fn() -> G),
 }
 
+impl<G: RuntimeGenesis> Clone for GenesisSource<G> {
+	fn clone(&self) -> Self {
+		match *self {
+			GenesisSource::File(ref path) => GenesisSource::File(path.clone()),
+			GenesisSource::Embedded(d) => GenesisSource::Embedded(d),
+			GenesisSource::Factory(f) => GenesisSource::Factory(f),
+		}
+	}
+}
+
 impl<G: RuntimeGenesis> GenesisSource<G> {
 	fn resolve(&self) -> Result<Genesis<G>, String> {
 		#[derive(Serialize, Deserialize)]
@@ -69,7 +79,7 @@ enum Genesis<G> {
 	Raw(HashMap<StorageKey, StorageData>),
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Clone)]
 #[serde(rename_all = "camelCase")]
 struct ChainSpecFile {
 	pub name: String,
@@ -85,6 +95,15 @@ pub struct ChainSpec<G: RuntimeGenesis> {
 	genesis: GenesisSource<G>,
 }
 
+impl<G: RuntimeGenesis> Clone for ChainSpec<G> {
+	fn clone(&self) -> Self {
+		ChainSpec {
+			spec: self.spec.clone(),
+			genesis: self.genesis.clone(),
+		}
+	}
+}
+
 impl<G: RuntimeGenesis> ChainSpec<G> {
 	pub fn boot_nodes(&self) -> &[String] {
 		&self.spec.boot_nodes

substrate/core/service/src/components.rs

@@ -19,11 +19,13 @@
 use std::fmt;
 use std::sync::Arc;
 use std::marker::PhantomData;
+use std::ops::Deref;
 use serde::{Serialize, de::DeserializeOwned};
+use tokio::runtime::TaskExecutor;
 use chain_spec::ChainSpec;
 use client_db;
 use client::{self, Client};
-use error;
+use {error, Service};
 use network::{self, OnDemand};
 use substrate_executor::{NativeExecutor, NativeExecutionDispatch};
 use transaction_pool::{self, Options as TransactionPoolOptions, Pool as TransactionPool};
@@ -83,6 +85,9 @@ pub type FactoryGenesis<F> = <F as ServiceFactory>::Genesis;
 /// `Block` type for a factory.
 pub type FactoryBlock<F> = <F as ServiceFactory>::Block;
 
+/// `Extrinsic` type for a factory.
+pub type FactoryExtrinsic<F> = <<F as ServiceFactory>::Block as BlockT>::Extrinsic;
+
 /// `Number` type for a factory.
 pub type FactoryBlockNumber<F> = <<FactoryBlock<F> as BlockT>::Header as HeaderT>::Number;
 
@@ -113,7 +118,7 @@ pub trait RuntimeGenesis: Serialize + DeserializeOwned + BuildStorage {}
 impl<T: Serialize + DeserializeOwned + BuildStorage> RuntimeGenesis for T {}
 
 /// A collection of types and methods to build a service on top of the substrate service.
-pub trait ServiceFactory: 'static {
+pub trait ServiceFactory: 'static + Sized {
 	/// Block type.
 	type Block: BlockT;
 	/// Extrinsic hash type.
@@ -123,13 +128,17 @@ pub trait ServiceFactory: 'static {
 	/// Chain runtime.
 	type RuntimeDispatch: NativeExecutionDispatch + Send + Sync + 'static;
 	/// Extrinsic pool backend type for the full client.
-	type FullTransactionPoolApi: transaction_pool::ChainApi<Hash=Self::ExtrinsicHash, Block=Self::Block> + Send + 'static;
+	type FullTransactionPoolApi: transaction_pool::ChainApi<Hash = Self::ExtrinsicHash, Block = Self::Block> + Send + 'static;
 	/// Extrinsic pool backend type for the light client.
-	type LightTransactionPoolApi: transaction_pool::ChainApi<Hash=Self::ExtrinsicHash, Block=Self::Block> + 'static;
+	type LightTransactionPoolApi: transaction_pool::ChainApi<Hash = Self::ExtrinsicHash, Block = Self::Block> + 'static;
 	/// Genesis configuration for the runtime.
 	type Genesis: RuntimeGenesis;
 	/// Other configuration for service members.
 	type Configuration: Default;
+	/// Extended full service type.
+	type FullService: Deref<Target = Service<FullComponents<Self>>> + Send + Sync + 'static;
+	/// Extended light service type.
+	type LightService: Deref<Target = Service<LightComponents<Self>>> + Send + Sync + 'static;
 
 	//TODO: replace these with a constructor trait. that TransactionPool implements.
 	/// Extrinsic pool constructor for the full client.
@@ -142,6 +151,13 @@ pub trait ServiceFactory: 'static {
 	/// Build network protocol.
 	fn build_network_protocol(config: &FactoryFullConfiguration<Self>)
 		-> Result<Self::NetworkProtocol, error::Error>;
+
+	/// Build full service.
+	fn new_full(config: FactoryFullConfiguration<Self>, executor: TaskExecutor)
+		-> Result<Self::FullService, error::Error>;
+	/// Build light service.
+	fn new_light(config: FactoryFullConfiguration<Self>, executor: TaskExecutor)
+		-> Result<Self::LightService, error::Error>;
 }
 
 /// A collection of types and function to generalise over full / light client type.
@@ -153,7 +169,10 @@ pub trait Components: 'static {
 	/// Client executor.
 	type Executor: 'static + client::CallExecutor<FactoryBlock<Self::Factory>, Blake2Hasher> + Send + Sync;
 	/// Extrinsic pool type.
-	type TransactionPoolApi: 'static + transaction_pool::ChainApi<Hash=<Self::Factory as ServiceFactory>::ExtrinsicHash, Block=FactoryBlock<Self::Factory>>;
+	type TransactionPoolApi: 'static + transaction_pool::ChainApi<
+		Hash = <Self::Factory as ServiceFactory>::ExtrinsicHash,
+		Block = FactoryBlock<Self::Factory>
+	>;
 
 	/// Create client.
 	fn build_client(
@@ -195,7 +214,13 @@ impl<Factory: ServiceFactory> Components for FullComponents<Factory> {
 			path: config.database_path.as_str().into(),
 			pruning: config.pruning.clone(),
 		};
-		Ok((Arc::new(client_db::new_client(db_settings, executor, &config.chain_spec, config.execution_strategy)?), None))
+		Ok((Arc::new(client_db::new_client(
+			db_settings,
+			executor,
+			&config.chain_spec,
+			config.block_execution_strategy,
+			config.api_execution_strategy,
+		)?), None))
 	}
 
 	fn build_transaction_pool(config: TransactionPoolOptions, client: Arc<ComponentClient<Self>>)

substrate/core/service/src/config.rs

@@ -28,6 +28,7 @@ use serde::{Serialize, de::DeserializeOwned};
 use target_info::Target;
 
 /// Service configuration.
+#[derive(Clone)]
 pub struct Configuration<C, G: Serialize + DeserializeOwned + BuildStorage> {
 	/// Implementation name
 	pub impl_name: &'static str,
@@ -53,12 +54,12 @@ pub struct Configuration<C, G: Serialize + DeserializeOwned + BuildStorage> {
 	pub chain_spec: ChainSpec<G>,
 	/// Custom configuration.
 	pub custom: C,
-	/// Telemetry server URL, optional - only `Some` if telemetry reporting is enabled
-	pub telemetry: Option<String>,
 	/// Node name.
 	pub name: String,
-	/// Execution strategy.
-	pub execution_strategy: ExecutionStrategy,
+	/// Block execution strategy.
+	pub block_execution_strategy: ExecutionStrategy,
+	/// Runtime API execution strategy.
+	pub api_execution_strategy: ExecutionStrategy,
 	/// RPC over HTTP binding address. `None` if disabled.
 	pub rpc_http: Option<SocketAddr>,
 	/// RPC over Websockets binding address. `None` if disabled.
@@ -83,9 +84,9 @@ impl<C: Default, G: Serialize + DeserializeOwned + BuildStorage> Configuration<C
 			database_path: Default::default(),
 			keys: Default::default(),
 			custom: Default::default(),
-			telemetry: Default::default(),
 			pruning: PruningMode::default(),
-			execution_strategy: ExecutionStrategy::Both,
+			block_execution_strategy: ExecutionStrategy::Both,
+			api_execution_strategy: ExecutionStrategy::Both,
 			rpc_http: None,
 			rpc_ws: None,
 			telemetry_url: None,

substrate/core/service/src/lib.rs

@@ -25,6 +25,7 @@ extern crate futures;
 extern crate exit_future;
 extern crate serde;
 extern crate serde_json;
+extern crate parking_lot;
 extern crate substrate_keystore as keystore;
 extern crate substrate_primitives as primitives;
 extern crate sr_primitives as runtime_primitives;
@@ -61,6 +62,7 @@ use std::net::SocketAddr;
 use std::sync::Arc;
 use std::collections::HashMap;
 use futures::prelude::*;
+use parking_lot::Mutex;
 use keystore::Store as Keystore;
 use client::BlockchainEvents;
 use runtime_primitives::traits::{Header, As};
@@ -81,7 +83,7 @@ pub use components::{ServiceFactory, FullBackend, FullExecutor, LightBackend,
 	ComponentBlock, FullClient, LightClient, FullComponents, LightComponents,
 	CodeExecutor, NetworkService, FactoryChainSpec, FactoryBlock,
 	FactoryFullConfiguration, RuntimeGenesis, FactoryGenesis,
-	ComponentExHash, ComponentExtrinsic,
+	ComponentExHash, ComponentExtrinsic, FactoryExtrinsic,
 };
 
 const DEFAULT_PROTOCOL_ID: &'static str = "sup";
@@ -95,7 +97,7 @@ pub struct Service<Components: components::Components> {
 	exit: ::exit_future::Exit,
 	signal: Option<Signal>,
 	_rpc_http: Option<rpc::HttpServer>,
-	_rpc_ws: Option<rpc::WsServer>,
+	_rpc_ws: Option<Mutex<rpc::WsServer>>, // WsServer is not `Sync`, but the service needs to be.
 	_telemetry: Option<tel::Telemetry>,
 }
 
@@ -186,12 +188,14 @@ impl<Components> Service<Components>
 
 		{
 			// block notifications
-			let network = network.clone();
+			let network = Arc::downgrade(&network);
 			let txpool = transaction_pool.clone();
 
 			let events = client.import_notification_stream()
 				.for_each(move |notification| {
-					network.on_block_imported(notification.hash, &notification.header);
+					if let Some(network) = network.upgrade() {
+						network.on_block_imported(notification.hash, &notification.header);
+					}
 					txpool.cull(&BlockId::hash(notification.hash))
 						.map_err(|e| warn!("Error removing extrinsics: {:?}", e))?;
 					Ok(())
@@ -203,11 +207,13 @@ impl<Components> Service<Components>
 
 		{
 			// extrinsic notifications
-			let network = network.clone();
+			let network = Arc::downgrade(&network);
 			let events = transaction_pool.import_notification_stream()
 				// TODO [ToDr] Consider throttling?
 				.for_each(move |_| {
-					network.trigger_repropagate();
+					if let Some(network) = network.upgrade() {
+						network.trigger_repropagate();
+					}
 					Ok(())
 				})
 				.select(exit.clone())
@@ -277,7 +283,7 @@ impl<Components> Service<Components>
 			keystore: keystore,
 			exit,
 			_rpc_http: rpc_http,
-			_rpc_ws: rpc_ws,
+			_rpc_ws: rpc_ws.map(Mutex::new),
 			_telemetry: telemetry,
 		})
 	}