Update Cumulus for Parachains V1 (#224)

* Start with something

* Whatever

* Update

* MOARE

* Make cumulus-network compile and tests work

* Update more and fixes

* More stuff

* More fixes

* Make collator build

* Make test almost work

* Remove contracts runtime

* More test work

* Make service compile

* Fix test-service

* Fix test client

* More fixes

* Fix collator test

* Fix network tests (again)

* Make everything compile, finally

* Fix tests

* Update to latest masters

* Remove ignore

* Switch to different branch in polkadot for now

* Update reference

* Make it compile with latest changes

* Update collator/src/lib.rs

Co-authored-by: Robert Habermeier <rphmeier@gmail.com>

* Update to latest upstream

* Update to latest master

* Fix test

Co-authored-by: Robert Habermeier <rphmeier@gmail.com>

cumulus/test/service/src/lib.rs

@@ -24,21 +24,17 @@ mod genesis;
 pub use chain_spec::*;
 pub use genesis::*;
 
-use ansi_term::Color;
 use core::future::Future;
-use cumulus_collator::CollatorBuilder;
-use cumulus_network::DelayedBlockAnnounceValidator;
+use cumulus_network::BlockAnnounceValidator;
 use cumulus_primitives::ParaId;
 use cumulus_service::{
-	prepare_node_config, start_full_node, StartCollatorParams, StartFullNodeParams,
+	prepare_node_config, start_collator, start_full_node, StartCollatorParams, StartFullNodeParams,
 };
 use cumulus_test_runtime::{NodeBlock as Block, RuntimeApi};
-use polkadot_primitives::v0::CollatorPair;
+use polkadot_primitives::v1::CollatorPair;
 use sc_client_api::execution_extensions::ExecutionStrategies;
-use sc_client_api::BlockBackend;
 use sc_executor::native_executor_instance;
 pub use sc_executor::NativeExecutor;
-use sc_informant::OutputFormat;
 use sc_network::{config::TransportConfig, multiaddr, NetworkService};
 use sc_service::{
 	config::{
@@ -48,8 +44,7 @@ use sc_service::{
 	BasePath, ChainSpec, Configuration, Error as ServiceError, PartialComponents, Role,
 	RpcHandlers, TFullBackend, TFullClient, TaskExecutor, TaskManager,
 };
-use sp_consensus::{BlockImport, Environment, Error as ConsensusError, Proposer};
-use sp_core::{crypto::Pair, H256};
+use sp_core::{Pair, H256};
 use sp_keyring::Sr25519Keyring;
 use sp_runtime::traits::BlakeTwo256;
 use sp_state_machine::BasicExternalities;
@@ -83,7 +78,7 @@ pub fn new_partial(
 > {
 	let inherent_data_providers = sp_inherents::InherentDataProviders::new();
 
-	let (client, backend, keystore, task_manager) =
+	let (client, backend, keystore_container, task_manager) =
 		sc_service::new_full_parts::<Block, RuntimeApi, RuntimeExecutor>(&config)?;
 	let client = Arc::new(client);
 
@@ -108,7 +103,7 @@ pub fn new_partial(
 		backend,
 		client,
 		import_queue,
-		keystore,
+		keystore_container,
 		task_manager,
 		transaction_pool,
 		inherent_data_providers,
@@ -119,96 +114,22 @@ pub fn new_partial(
 	Ok(params)
 }
 
-/// Start a test collator node for a parachain.
-///
-/// A collator is similar to a validator in a normal blockchain.
-/// It is responsible for producing blocks and sending the blocks to a
-/// parachain validator for validation and inclusion into the relay chain.
-pub fn start_test_collator<'a, PF, BI, BS>(
-	StartCollatorParams {
-		para_id,
-		proposer_factory,
-		inherent_data_providers,
-		block_import,
-		block_status,
-		announce_block,
-		client,
-		block_announce_validator,
-		task_manager,
-		polkadot_config,
-		collator_key,
-	}: StartCollatorParams<'a, Block, PF, BI, BS, TFullClient<Block, RuntimeApi, RuntimeExecutor>>,
-) -> sc_service::error::Result<()>
-where
-	PF: Environment<Block> + Send + 'static,
-	BI: BlockImport<
-			Block,
-			Error = ConsensusError,
-			Transaction = <PF::Proposer as Proposer<Block>>::Transaction,
-		> + Send
-		+ Sync
-		+ 'static,
-	BS: BlockBackend<Block> + Send + Sync + 'static,
-{
-	let builder = CollatorBuilder::new(
-		proposer_factory,
-		inherent_data_providers,
-		block_import,
-		block_status,
-		para_id,
-		client,
-		announce_block,
-		block_announce_validator,
-	);
-
-	let (polkadot_future, polkadot_task_manager) = {
-		let (task_manager, client, handles, _network, _rpc_handlers) =
-			polkadot_test_service::polkadot_test_new_full(
-				polkadot_config,
-				Some((collator_key.public(), para_id)),
-				None,
-				false,
-				6000,
-			)?;
-
-		let test_client = polkadot_test_service::TestClient(client);
-
-		let future = polkadot_collator::build_collator_service(
-			task_manager.spawn_handle(),
-			handles,
-			test_client,
-			para_id,
-			collator_key,
-			builder,
-		)?;
-
-		(future, task_manager)
-	};
-
-	task_manager
-		.spawn_essential_handle()
-		.spawn("polkadot", polkadot_future);
-
-	task_manager.add_child(polkadot_task_manager);
-
-	Ok(())
-}
-
 /// Start a node with the given parachain `Configuration` and relay chain `Configuration`.
 ///
 /// This is the actual implementation that is abstract over the executor and the runtime api.
-fn start_node_impl<RB>(
+#[sc_cli::prefix_logs_with(parachain_config.network.node_name.as_str())]
+async fn start_node_impl<RB>(
 	parachain_config: Configuration,
-	collator_key: Arc<CollatorPair>,
-	mut polkadot_config: polkadot_collator::Configuration,
+	collator_key: CollatorPair,
+	polkadot_config: Configuration,
 	para_id: ParaId,
-	validator: bool,
+	is_collator: bool,
 	rpc_ext_builder: RB,
 ) -> sc_service::error::Result<(
 	TaskManager,
 	Arc<TFullClient<Block, RuntimeApi, RuntimeExecutor>>,
 	Arc<NetworkService<Block, H256>>,
-	Arc<RpcHandlers>,
+	RpcHandlers,
 )>
 where
 	RB: Fn(
@@ -223,32 +144,30 @@ where
 
 	let mut parachain_config = prepare_node_config(parachain_config);
 
-	parachain_config.informant_output_format = OutputFormat {
-		enable_color: true,
-		prefix: format!("[{}] ", Color::Yellow.bold().paint("Parachain")),
-	};
-	polkadot_config.informant_output_format = OutputFormat {
-		enable_color: true,
-		prefix: format!("[{}] ", Color::Blue.bold().paint("Relaychain")),
-	};
-
 	let params = new_partial(&mut parachain_config)?;
 	params
 		.inherent_data_providers
 		.register_provider(sp_timestamp::InherentDataProvider)
-		.unwrap();
+		.expect("Registers timestamp inherent data provider.");
+
+	let transaction_pool = params.transaction_pool.clone();
+	let mut task_manager = params.task_manager;
+
+	let polkadot_full_node = polkadot_test_service::new_full(
+		polkadot_config,
+		polkadot_service::IsCollator::Yes(collator_key.public()),
+	)?;
 
 	let client = params.client.clone();
 	let backend = params.backend.clone();
-	let block_announce_validator = DelayedBlockAnnounceValidator::new();
-	let block_announce_validator_builder = {
-		let block_announce_validator = block_announce_validator.clone();
-		move |_| Box::new(block_announce_validator) as Box<_>
-	};
+	let block_announce_validator = BlockAnnounceValidator::new(
+		polkadot_full_node.client.clone(),
+		para_id,
+		Box::new(polkadot_full_node.network.clone()),
+	);
+	let block_announce_validator_builder = move |_| Box::new(block_announce_validator) as Box<_>;
 
 	let prometheus_registry = parachain_config.prometheus_registry().cloned();
-	let transaction_pool = params.transaction_pool.clone();
-	let mut task_manager = params.task_manager;
 	let import_queue = params.import_queue;
 	let (network, network_status_sinks, system_rpc_tx, start_network) =
 		sc_service::build_network(sc_service::BuildNetworkParams {
@@ -266,7 +185,7 @@ where
 	let rpc_extensions_builder = {
 		let client = client.clone();
 
-		Box::new(move |_deny_unsafe| rpc_ext_builder(client.clone()))
+		Box::new(move |_, _| rpc_ext_builder(client.clone()))
 	};
 
 	let rpc_handlers = sc_service::spawn_tasks(sc_service::SpawnTasksParams {
@@ -278,7 +197,7 @@ where
 		task_manager: &mut task_manager,
 		telemetry_connection_sinks: Default::default(),
 		config: parachain_config,
-		keystore: params.keystore,
+		keystore: params.keystore_container.sync_keystore(),
 		backend,
 		network: network.clone(),
 		network_status_sinks,
@@ -290,37 +209,38 @@ where
 		Arc::new(move |hash, data| network.announce_block(hash, data))
 	};
 
-	if validator {
+	let polkadot_full_node = polkadot_full_node.with_client(polkadot_test_service::TestClient);
+	if is_collator {
 		let proposer_factory = sc_basic_authorship::ProposerFactory::new(
+			task_manager.spawn_handle(),
 			client.clone(),
 			transaction_pool,
 			prometheus_registry.as_ref(),
 		);
 
 		let params = StartCollatorParams {
-			para_id,
-			block_import: client.clone(),
 			proposer_factory,
 			inherent_data_providers: params.inherent_data_providers,
+			backend: params.backend,
+			block_import: client.clone(),
 			block_status: client.clone(),
 			announce_block,
 			client: client.clone(),
-			block_announce_validator,
+			spawner: task_manager.spawn_handle(),
 			task_manager: &mut task_manager,
-			polkadot_config,
+			para_id,
 			collator_key,
+			polkadot_full_node,
 		};
 
-		start_test_collator(params)?;
+		start_collator(params).await?;
 	} else {
 		let params = StartFullNodeParams {
 			client: client.clone(),
 			announce_block,
-			polkadot_config,
-			collator_key,
-			block_announce_validator,
 			task_manager: &mut task_manager,
 			para_id,
+			polkadot_full_node,
 		};
 
 		start_full_node(params)?;
@@ -343,14 +263,14 @@ pub struct CumulusTestNode {
 	/// to other nodes.
 	pub addr: MultiaddrWithPeerId,
 	/// RPCHandlers to make RPC queries.
-	pub rpc_handlers: Arc<RpcHandlers>,
+	pub rpc_handlers: RpcHandlers,
 }
 
 /// Run a Cumulus test node using the Cumulus test runtime. The node will be using an in-memory
 /// socket, therefore you need to provide boot nodes if you want it to be connected to other nodes.
 /// The `storage_update_func` can be used to make adjustements to the runtime before the node
 /// starts.
-pub fn run_test_node(
+pub async fn run_test_node(
 	task_executor: TaskExecutor,
 	key: Sr25519Keyring,
 	parachain_storage_update_func: impl Fn(),
@@ -358,32 +278,39 @@ pub fn run_test_node(
 	parachain_boot_nodes: Vec<MultiaddrWithPeerId>,
 	polkadot_boot_nodes: Vec<MultiaddrWithPeerId>,
 	para_id: ParaId,
-	validator: bool,
+	is_collator: bool,
 ) -> CumulusTestNode {
-	let collator_key = Arc::new(sp_core::Pair::generate().0);
+	let collator_key = CollatorPair::generate().0;
 	let parachain_config = node_config(
 		parachain_storage_update_func,
 		task_executor.clone(),
 		key,
 		parachain_boot_nodes,
 		para_id,
+		is_collator,
 	)
 	.expect("could not generate Configuration");
-	let polkadot_config = polkadot_test_service::node_config(
+	let mut polkadot_config = polkadot_test_service::node_config(
 		polkadot_storage_update_func,
 		task_executor.clone(),
 		key,
 		polkadot_boot_nodes,
+		false,
 	);
+
+	polkadot_config.network.node_name =
+		format!("{} (relay chain)", polkadot_config.network.node_name);
+
 	let multiaddr = parachain_config.network.listen_addresses[0].clone();
-	let (task_manager, client, network, rpc_handlers) = start_node_impl::<_>(
+	let (task_manager, client, network, rpc_handlers) = start_node_impl(
 		parachain_config,
 		collator_key,
 		polkadot_config,
 		para_id,
-		validator,
+		is_collator,
 		|_| Default::default(),
 	)
+	.await
 	.expect("could not create Cumulus test service");
 
 	let peer_id = network.local_peer_id().clone();
@@ -407,11 +334,16 @@ pub fn node_config(
 	key: Sr25519Keyring,
 	boot_nodes: Vec<MultiaddrWithPeerId>,
 	para_id: ParaId,
+	is_collator: bool,
 ) -> Result<Configuration, ServiceError> {
 	let base_path = BasePath::new_temp_dir()?;
 	let root = base_path.path().to_path_buf();
-	let role = Role::Authority {
-		sentry_nodes: Vec::new(),
+	let role = if is_collator {
+		Role::Authority {
+			sentry_nodes: Vec::new(),
+		}
+	} else {
+		Role::Full
 	};
 	let key_seed = key.to_seed();
 	let mut spec = Box::new(chain_spec::get_chain_spec(para_id));
@@ -425,19 +357,15 @@ pub fn node_config(
 	spec.set_storage(storage);
 
 	let mut network_config = NetworkConfiguration::new(
-		format!("Cumulus Test Node for: {}", key_seed),
+		format!("{} (parachain)", key_seed.to_string()),
 		"network/test/0.1",
 		Default::default(),
 		None,
 	);
-	let informant_output_format = OutputFormat {
-		enable_color: false,
-		prefix: format!("[{}] ", key_seed),
-	};
 
 	network_config.boot_nodes = boot_nodes;
 
-	network_config.allow_non_globals_in_dht = false;
+	network_config.allow_non_globals_in_dht = true;
 
 	network_config
 		.listen_addresses
@@ -452,10 +380,7 @@ pub fn node_config(
 		task_executor,
 		transaction_pool: Default::default(),
 		network: network_config,
-		keystore: KeystoreConfig::Path {
-			path: root.join("key"),
-			password: None,
-		},
+		keystore: KeystoreConfig::InMemory,
 		database: DatabaseConfig::RocksDb {
 			path: root.join("db"),
 			cache_size: 128,
@@ -495,7 +420,8 @@ pub fn node_config(
 		max_runtime_instances: 8,
 		announce_block: true,
 		base_path: Some(base_path),
-		informant_output_format,
+		informant_output_format: Default::default(),
+		wasm_runtime_overrides: None,
 	})
 }