Uniformize tests (#220)

* Initial commit

Forked at: 56753b7717
Parent branch: origin/master

* Copy runtime module from rococo

Forked at: 56753b7717
Parent branch: origin/master

* Also copy dependencies pallets and primitives

Forked at: 56753b7717
Parent branch: origin/master

* WIP

Forked at: 56753b7717
Parent branch: origin/master

* WIP

Forked at: 56753b7717
Parent branch: origin/master

* test-service

* Move integration test

* CLEANUP

Forked at: 56753b7717
Parent branch: origin/master

* Not sure what went wrong...

* WIP

Forked at: 56753b7717
Parent branch: origin/master

* WIP

Forked at: 56753b7717
Parent branch: origin/master

* CLEANUP

Forked at: 56753b7717
Parent branch: origin/master

* fmt

* CLEANUP

Forked at: 56753b7717
Parent branch: origin/master

* CLEANUP

Forked at: 56753b7717
Parent branch: origin/master

* Remove pallet contracts (not used)

* Remove pallet parachain-info and token-dealer (not used)

* Sort dependencies alphabetically

* CLEANUP

Forked at: 56753b7717
Parent branch: origin/master

* CumulusTestNode for testing

* Speed up block generation

* Fix improper shutdown

* rustfmt

* runtime: replace const by storage

* Fix for previous commit

* Remove some generics

* Move generate_genesis_state to cumulus-primitives

* fmt

* Remove message_example

* fixup! Remove message_example

* WIP

Forked at: 56753b7717
Parent branch: origin/master

* Half the solution to previous commit :(

* Revert "Fix for previous commit"

This reverts commit 60010bab6797487093ac8c790b3a536f7ca0895b.

* Revert "runtime: replace const by storage"

This reverts commit c64b3a46f0325a98922015e0cbf3570e2e431774.

Not working for some reason...

* Use helper

Forked at: 56753b7717
Parent branch: origin/master

* WIP

Forked at: 56753b7717
Parent branch: origin/master

* Remove test-primitives

* Revert "Half the solution to previous commit :("

This reverts commit 9a8f89f9f06252198e6405057043c6b313f1aea4.

* Revert "Revert "Half the solution to previous commit :(""

This reverts commit 6a93f0f09d74ccdc3738dd78a777c483427c03ce.

* Test with some extra extrinsics

* WIP

Forked at: 56753b7717
Parent branch: origin/master

* CLEANUP

Forked at: 56753b7717
Parent branch: origin/master

* WIP

Forked at: 56753b7717
Parent branch: origin/master

* WIP

Forked at: 56753b7717
Parent branch: origin/master

* WIP

Forked at: 56753b7717
Parent branch: origin/master

* WIP

Forked at: 56753b7717
Parent branch: origin/master

* WIP

Forked at: 56753b7717
Parent branch: origin/master

* WIP

Forked at: 56753b7717
Parent branch: origin/master

* WIP

Forked at: 56753b7717
Parent branch: origin/master

* CLEANUP

Forked at: 56753b7717
Parent branch: origin/master

* Remove message broker

cumulus/rococo-parachains/Cargo.toml

@@ -41,7 +41,6 @@ sc-executor = { git = "https://github.com/paritytech/substrate", branch = "rococ
 sc-service = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
 sc-transaction-pool = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
 sp-transaction-pool = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
-sc-network = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
 sc-basic-authorship = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch", version = "0.8.0-rc5" }
 sp-timestamp = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
 sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
@@ -70,7 +69,6 @@ polkadot-primitives = { git = "https://github.com/paritytech/polkadot", branch =
 polkadot-collator = { git = "https://github.com/paritytech/polkadot", branch = "rococo-branch" }
 polkadot-service = { git = "https://github.com/paritytech/polkadot", branch = "rococo-branch" }
 polkadot-cli = { git = "https://github.com/paritytech/polkadot", branch = "rococo-branch" }
-polkadot-test-service = { git = "https://github.com/paritytech/polkadot", branch = "rococo-branch" }
 polkadot-parachain = { git = "https://github.com/paritytech/polkadot", branch = "rococo-branch" }
 
 [build-dependencies]
@@ -79,17 +77,3 @@ substrate-build-script-utils = { git = "https://github.com/paritytech/substrate"
 [dev-dependencies]
 assert_cmd = "0.12"
 nix = "0.17"
-rand = "0.7.3"
-tokio = { version = "0.2.21", features = ["macros"] }
-
-# Polkadot dependencies
-polkadot-runtime-common = { git = "https://github.com/paritytech/polkadot", branch = "rococo-branch" }
-polkadot-test-runtime = { git = "https://github.com/paritytech/polkadot", branch = "rococo-branch" }
-polkadot-test-runtime-client = { git = "https://github.com/paritytech/polkadot", branch = "rococo-branch" }
-polkadot-test-service = { git = "https://github.com/paritytech/polkadot", branch = "rococo-branch" }
-
-# Substrate dependencies
-pallet-sudo = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
-substrate-test-client = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
-substrate-test-runtime-client = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
-substrate-test-utils = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }

cumulus/rococo-parachains/src/command.rs

@@ -19,7 +19,7 @@ use crate::{
 	cli::{Cli, RelayChainCli, Subcommand},
 };
 use codec::Encode;
-use cumulus_primitives::ParaId;
+use cumulus_primitives::{genesis::generate_genesis_block, ParaId};
 use log::info;
 use parachain_runtime::Block;
 use polkadot_parachain::primitives::AccountIdConversion;
@@ -29,7 +29,7 @@ use sc_cli::{
 };
 use sc_service::config::{BasePath, PrometheusConfig};
 use sp_core::hexdisplay::HexDisplay;
-use sp_runtime::traits::{Block as BlockT, Hash as HashT, Header as HeaderT, Zero};
+use sp_runtime::traits::Block as BlockT;
 use std::{io::Write, net::SocketAddr, sync::Arc};
 
 fn load_spec(
@@ -134,34 +134,6 @@ impl SubstrateCli for RelayChainCli {
 	}
 }
 
-pub fn generate_genesis_state(chain_spec: &Box<dyn sc_service::ChainSpec>) -> Result<Block> {
-	let storage = chain_spec.build_storage()?;
-
-	let child_roots = storage.children_default.iter().map(|(sk, child_content)| {
-		let state_root = <<<Block as BlockT>::Header as HeaderT>::Hashing as HashT>::trie_root(
-			child_content.data.clone().into_iter().collect(),
-		);
-		(sk.clone(), state_root.encode())
-	});
-	let state_root = <<<Block as BlockT>::Header as HeaderT>::Hashing as HashT>::trie_root(
-		storage.top.clone().into_iter().chain(child_roots).collect(),
-	);
-
-	let extrinsics_root =
-		<<<Block as BlockT>::Header as HeaderT>::Hashing as HashT>::trie_root(Vec::new());
-
-	Ok(Block::new(
-		<<Block as BlockT>::Header as HeaderT>::new(
-			Zero::zero(),
-			extrinsics_root,
-			state_root,
-			Default::default(),
-			Default::default(),
-		),
-		Default::default(),
-	))
-}
-
 fn extract_genesis_wasm(chain_spec: &Box<dyn sc_service::ChainSpec>) -> Result<Vec<u8>> {
 	let mut storage = chain_spec.build_storage()?;
 
@@ -216,7 +188,7 @@ pub fn run() -> Result<()> {
 		Some(Subcommand::ExportGenesisState(params)) => {
 			sc_cli::init_logger("");
 
-			let block = generate_genesis_state(&load_spec(
+			let block: Block = generate_genesis_block(&load_spec(
 				&params.chain.clone().unwrap_or_default(),
 				params.parachain_id.into(),
 			)?)?;
@@ -268,8 +240,8 @@ pub fn run() -> Result<()> {
 				let parachain_account =
 					AccountIdConversion::<polkadot_primitives::v0::AccountId>::into_account(&id);
 
-				let block =
-					generate_genesis_state(&config.chain_spec).map_err(|e| format!("{:?}", e))?;
+				let block: Block =
+					generate_genesis_block(&config.chain_spec).map_err(|e| format!("{:?}", e))?;
 				let genesis_state = format!("0x{:?}", HexDisplay::from(&block.header().encode()));
 
 				let task_executor = config.task_executor.clone();
@@ -293,7 +265,6 @@ pub fn run() -> Result<()> {
 						polkadot_config,
 						id,
 						collator,
-						false,
 					)
 				} else {
 					crate::service::start_node(
@@ -302,7 +273,6 @@ pub fn run() -> Result<()> {
 						polkadot_config,
 						id,
 						collator,
-						false,
 					)
 					.map(|r| r.0)
 				}

cumulus/rococo-parachains/src/integration_test.rs

@@ -1,215 +0,0 @@
-// Copyright 2020 Parity Technologies (UK) Ltd.
-// This file is part of Substrate.
-
-// Substrate is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Substrate is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
-
-use codec::Encode;
-use futures::future;
-use polkadot_primitives::v0::{Id as ParaId, Info, Scheduling};
-use polkadot_runtime_common::registrar;
-use polkadot_test_runtime_client::Sr25519Keyring;
-use sc_chain_spec::ChainSpec;
-use sc_client_api::execution_extensions::ExecutionStrategies;
-use sc_informant::OutputFormat;
-use sc_network::{config::TransportConfig, multiaddr};
-use sc_service::{
-	config::{
-		DatabaseConfig, KeystoreConfig, MultiaddrWithPeerId, NetworkConfiguration,
-		OffchainWorkerConfig, PruningMode, WasmExecutionMethod,
-	},
-	BasePath, Configuration, Error as ServiceError, Role, TaskExecutor,
-};
-use sp_api::BlockT;
-use std::sync::Arc;
-use substrate_test_client::BlockchainEventsExt;
-use substrate_test_runtime_client::AccountKeyring::*;
-
-#[substrate_test_utils::test]
-#[ignore]
-async fn integration_test(task_executor: TaskExecutor) {
-	let para_id = ParaId::from(100);
-
-	// generate parachain spec
-	let spec = Box::new(crate::chain_spec::get_chain_spec(para_id));
-
-	// start alice
-	let alice = polkadot_test_service::run_test_node(task_executor.clone(), Alice, || {}, vec![]);
-
-	// start bob
-	let bob = polkadot_test_service::run_test_node(
-		task_executor.clone(),
-		Bob,
-		|| {},
-		vec![alice.addr.clone()],
-	);
-
-	// ensure alice and bob can produce blocks
-	future::join(alice.wait_for_blocks(2), bob.wait_for_blocks(2)).await;
-
-	// export genesis state
-	let block = crate::command::generate_genesis_state(&(spec.clone() as Box<_>)).unwrap();
-	let genesis_state = block.header().encode();
-
-	// create and sign transaction to register parachain
-	let function = polkadot_test_runtime::Call::Sudo(pallet_sudo::Call::sudo(Box::new(
-		polkadot_test_runtime::Call::Registrar(registrar::Call::register_para(
-			para_id,
-			Info {
-				scheduling: Scheduling::Always,
-			},
-			parachain_runtime::WASM_BINARY
-				.expect("You need to build the WASM binary to run this test!")
-				.to_vec()
-				.into(),
-			genesis_state.into(),
-		)),
-	)));
-
-	// register parachain
-	let _ = alice.call_function(function, Alice).await.unwrap();
-
-	// run cumulus charlie (a validator)
-	let key = Arc::new(sp_core::Pair::generate().0);
-	let polkadot_config = polkadot_test_service::node_config(
-		|| {},
-		task_executor.clone(),
-		Charlie,
-		vec![alice.addr.clone(), bob.addr.clone()],
-	);
-	let charlie_config =
-		parachain_config(task_executor.clone(), Charlie, vec![], spec.clone()).unwrap();
-	let multiaddr = charlie_config.network.listen_addresses[0].clone();
-	let (charlie_task_manager, charlie_client, charlie_network) =
-		crate::service::start_node(charlie_config, key, polkadot_config, para_id, true, true)
-			.unwrap();
-	charlie_client.wait_for_blocks(4).await;
-	let peer_id = charlie_network.local_peer_id().clone();
-	let charlie_addr = MultiaddrWithPeerId { multiaddr, peer_id };
-
-	// run cumulus dave (not a validator)
-	//
-	// a collator running in non-validator mode should be able to sync blocks from the tip of the
-	// parachain
-	let key = Arc::new(sp_core::Pair::generate().0);
-	let polkadot_config = polkadot_test_service::node_config(
-		|| {},
-		task_executor.clone(),
-		Dave,
-		vec![alice.addr.clone(), bob.addr.clone()],
-	);
-	let dave_config = parachain_config(
-		task_executor.clone(),
-		Dave,
-		vec![charlie_addr],
-		spec.clone(),
-	)
-	.unwrap();
-	let (dave_task_manager, dave_client, _dave_network) =
-		crate::service::start_node(dave_config, key, polkadot_config, para_id, false, true)
-			.unwrap();
-	dave_client.wait_for_blocks(4).await;
-
-	alice.task_manager.clean_shutdown();
-	bob.task_manager.clean_shutdown();
-	charlie_task_manager.clean_shutdown();
-	dave_task_manager.clean_shutdown();
-}
-
-pub fn parachain_config(
-	task_executor: TaskExecutor,
-	key: Sr25519Keyring,
-	boot_nodes: Vec<MultiaddrWithPeerId>,
-	spec: Box<dyn ChainSpec>,
-) -> Result<Configuration, ServiceError> {
-	let base_path = BasePath::new_temp_dir()?;
-	let root = base_path.path().to_path_buf();
-	let role = Role::Authority {
-		sentry_nodes: Vec::new(),
-	};
-	let key_seed = key.to_seed();
-
-	let mut network_config = NetworkConfiguration::new(
-		format!("Cumulus Test Node for: {}", key_seed),
-		"network/test/0.1",
-		Default::default(),
-		None,
-	);
-	let informant_output_format = OutputFormat {
-		enable_color: false,
-		prefix: format!("[{}] ", key_seed),
-	};
-
-	network_config.boot_nodes = boot_nodes;
-
-	network_config.allow_non_globals_in_dht = false;
-
-	network_config
-		.listen_addresses
-		.push(multiaddr::Protocol::Memory(rand::random()).into());
-
-	network_config.transport = TransportConfig::MemoryOnly;
-
-	Ok(Configuration {
-		impl_name: "cumulus-test-node".to_string(),
-		impl_version: "0.1".to_string(),
-		role,
-		task_executor,
-		transaction_pool: Default::default(),
-		network: network_config,
-		keystore: KeystoreConfig::Path {
-			path: root.join("key"),
-			password: None,
-		},
-		database: DatabaseConfig::RocksDb {
-			path: root.join("db"),
-			cache_size: 128,
-		},
-		state_cache_size: 67108864,
-		state_cache_child_ratio: None,
-		pruning: PruningMode::ArchiveAll,
-		chain_spec: spec,
-		wasm_method: WasmExecutionMethod::Interpreted,
-		// NOTE: we enforce the use of the native runtime to make the errors more debuggable
-		execution_strategies: ExecutionStrategies {
-			syncing: sc_client_api::ExecutionStrategy::NativeWhenPossible,
-			importing: sc_client_api::ExecutionStrategy::NativeWhenPossible,
-			block_construction: sc_client_api::ExecutionStrategy::NativeWhenPossible,
-			offchain_worker: sc_client_api::ExecutionStrategy::NativeWhenPossible,
-			other: sc_client_api::ExecutionStrategy::NativeWhenPossible,
-		},
-		rpc_http: None,
-		rpc_ws: None,
-		rpc_ipc: None,
-		rpc_ws_max_connections: None,
-		rpc_cors: None,
-		rpc_methods: Default::default(),
-		prometheus_config: None,
-		telemetry_endpoints: None,
-		telemetry_external_transport: None,
-		default_heap_pages: None,
-		offchain_worker: OffchainWorkerConfig {
-			enabled: true,
-			indexing_enabled: false,
-		},
-		force_authoring: false,
-		disable_grandpa: false,
-		dev_key_seed: Some(key_seed),
-		tracing_targets: None,
-		tracing_receiver: Default::default(),
-		max_runtime_instances: 8,
-		announce_block: true,
-		base_path: Some(base_path),
-		informant_output_format,
-	})
-}

cumulus/rococo-parachains/src/main.rs

@@ -24,8 +24,6 @@ mod chain_spec;
 mod service;
 mod cli;
 mod command;
-#[cfg(test)]
-mod integration_test;
 
 fn main() -> sc_cli::Result<()> {
 	command::run()

cumulus/rococo-parachains/src/service.rs

@@ -15,24 +15,18 @@
 // along with Cumulus.  If not, see <http://www.gnu.org/licenses/>.
 
 use ansi_term::Color;
-use cumulus_collator::CollatorBuilder;
 use cumulus_network::DelayedBlockAnnounceValidator;
 use cumulus_service::{
 	prepare_node_config, start_collator, start_full_node, StartCollatorParams, StartFullNodeParams,
 };
 use polkadot_primitives::v0::CollatorPair;
 use rococo_parachain_primitives::Block;
-use sc_client_api::{Backend as BackendT, BlockBackend, Finalizer, UsageProvider};
 use sc_executor::native_executor_instance;
 pub use sc_executor::NativeExecutor;
 use sc_informant::OutputFormat;
-use sc_network::NetworkService;
 use sc_service::{Configuration, PartialComponents, Role, TFullBackend, TFullClient, TaskManager};
 use sp_api::ConstructRuntimeApi;
-use sp_blockchain::HeaderBackend;
-use sp_consensus::{BlockImport, Environment, Error as ConsensusError, Proposer};
-use sp_core::{crypto::Pair, H256};
-use sp_runtime::traits::{BlakeTwo256, Block as BlockT};
+use sp_runtime::traits::BlakeTwo256;
 use sp_trie::PrefixedMemoryDB;
 use std::sync::Arc;
 
@@ -122,91 +116,6 @@ where
 	Ok(params)
 }
 
-/// Start a test collator node for a parachain.
-///
-/// A collator is similar to a validator in a normal blockchain.
-/// It is responsible for producing blocks and sending the blocks to a
-/// parachain validator for validation and inclusion into the relay chain.
-pub fn start_test_collator<'a, Block, PF, BI, BS, Client, Backend>(
-	StartCollatorParams {
-		para_id,
-		proposer_factory,
-		inherent_data_providers,
-		block_import,
-		block_status,
-		announce_block,
-		client,
-		block_announce_validator,
-		task_manager,
-		polkadot_config,
-		collator_key,
-	}: StartCollatorParams<'a, Block, PF, BI, BS, Client>,
-) -> sc_service::error::Result<()>
-where
-	Block: BlockT,
-	PF: Environment<Block> + Send + 'static,
-	BI: BlockImport<
-			Block,
-			Error = ConsensusError,
-			Transaction = <PF::Proposer as Proposer<Block>>::Transaction,
-		> + Send
-		+ Sync
-		+ 'static,
-	BS: BlockBackend<Block> + Send + Sync + 'static,
-	Client: Finalizer<Block, Backend>
-		+ UsageProvider<Block>
-		+ HeaderBackend<Block>
-		+ Send
-		+ Sync
-		+ BlockBackend<Block>
-		+ 'static,
-	for<'b> &'b Client: BlockImport<Block>,
-	Backend: BackendT<Block> + 'static,
-{
-	let builder = CollatorBuilder::new(
-		proposer_factory,
-		inherent_data_providers,
-		block_import,
-		block_status,
-		para_id,
-		client,
-		announce_block,
-		block_announce_validator,
-	);
-
-	let (polkadot_future, polkadot_task_manager) = {
-		let (task_manager, client, handles, _network, _rpc_handlers) =
-			polkadot_test_service::polkadot_test_new_full(
-				polkadot_config,
-				Some((collator_key.public(), para_id)),
-				None,
-				false,
-				6000,
-			)?;
-
-		let test_client = polkadot_test_service::TestClient(client);
-
-		let future = polkadot_collator::build_collator_service(
-			task_manager.spawn_handle(),
-			handles,
-			test_client,
-			para_id,
-			collator_key,
-			builder,
-		)?;
-
-		(future, task_manager)
-	};
-
-	task_manager
-		.spawn_essential_handle()
-		.spawn("polkadot", polkadot_future);
-
-	task_manager.add_child(polkadot_task_manager);
-
-	Ok(())
-}
-
 /// Start a node with the given parachain `Configuration` and relay chain `Configuration`.
 ///
 /// This is the actual implementation that is abstract over the executor and the runtime api.
@@ -217,12 +126,7 @@ fn start_node_impl<RuntimeApi, Executor, RB>(
 	id: polkadot_primitives::v0::Id,
 	validator: bool,
 	rpc_ext_builder: RB,
-	test: bool,
-) -> sc_service::error::Result<(
-	TaskManager,
-	Arc<TFullClient<Block, RuntimeApi, Executor>>,
-	Arc<NetworkService<Block, H256>>,
-)>
+) -> sc_service::error::Result<(TaskManager, Arc<TFullClient<Block, RuntimeApi, Executor>>)>
 where
 	RuntimeApi: ConstructRuntimeApi<Block, TFullClient<Block, RuntimeApi, Executor>>
 		+ Send
@@ -339,11 +243,7 @@ where
 			collator_key,
 		};
 
-		if test {
-			start_test_collator(params)?;
-		} else {
-			start_collator(params)?;
-		}
+		start_collator(params)?;
 	} else {
 		let params = StartFullNodeParams {
 			client: client.clone(),
@@ -360,7 +260,7 @@ where
 
 	start_network.start_network();
 
-	Ok((task_manager, client, network))
+	Ok((task_manager, client))
 }
 
 /// Start a normal parachain node.
@@ -370,11 +270,9 @@ pub fn start_node(
 	polkadot_config: polkadot_collator::Configuration,
 	id: polkadot_primitives::v0::Id,
 	validator: bool,
-	test: bool,
 ) -> sc_service::error::Result<(
 	TaskManager,
 	Arc<TFullClient<Block, parachain_runtime::RuntimeApi, RuntimeExecutor>>,
-	Arc<NetworkService<Block, H256>>,
 )> {
 	start_node_impl::<parachain_runtime::RuntimeApi, RuntimeExecutor, _>(
 		parachain_config,
@@ -383,7 +281,6 @@ pub fn start_node(
 		id,
 		validator,
 		|_| Default::default(),
-		test,
 	)
 }
 
@@ -394,7 +291,6 @@ pub fn start_contracts_node(
 	polkadot_config: polkadot_collator::Configuration,
 	id: polkadot_primitives::v0::Id,
 	validator: bool,
-	test: bool,
 ) -> sc_service::error::Result<TaskManager> {
 	start_node_impl::<parachain_contracts_runtime::RuntimeApi, ContractsRuntimeExecutor, _>(
 		parachain_config,
@@ -409,7 +305,6 @@ pub fn start_contracts_node(
 			io.extend_with(ContractsApi::to_delegate(Contracts::new(client)));
 			io
 		},
-		test,
 	)
 	.map(|r| r.0)
 }