Update Cumulus for Parachains V1 (#224)

* Start with something

* Whatever

* Update

* MOARE

* Make cumulus-network compile and tests work

* Update more and fixes

* More stuff

* More fixes

* Make collator build

* Make test almost work

* Remove contracts runtime

* More test work

* Make service compile

* Fix test-service

* Fix test client

* More fixes

* Fix collator test

* Fix network tests (again)

* Make everything compile, finally

* Fix tests

* Update to latest masters

* Remove ignore

* Switch to different branch in polkadot for now

* Update reference

* Make it compile with latest changes

* Update collator/src/lib.rs

Co-authored-by: Robert Habermeier <rphmeier@gmail.com>

* Update to latest upstream

* Update to latest master

* Fix test

Co-authored-by: Robert Habermeier <rphmeier@gmail.com>

cumulus/rococo-parachains/src/chain_spec.rs

@@ -26,10 +26,6 @@ use sp_runtime::traits::{IdentifyAccount, Verify};
 /// Specialized `ChainSpec` for the normal parachain runtime.
 pub type ChainSpec = sc_service::GenericChainSpec<parachain_runtime::GenesisConfig, Extensions>;
 
-/// Specialized `ChainSpec` for the contracts parachain runtime.
-pub type ContractsChainSpec =
-	sc_service::GenericChainSpec<parachain_contracts_runtime::GenesisConfig, Extensions>;
-
 /// Helper function to generate a crypto pair from seed
 pub fn get_from_seed<TPublic: Public>(seed: &str) -> <TPublic::Pair as Pair>::Public {
 	TPublic::Pair::from_string(&format!("//{}", seed), None)
@@ -100,42 +96,6 @@ pub fn get_chain_spec(id: ParaId) -> ChainSpec {
 	)
 }
 
-pub fn get_contracts_chain_spec(id: ParaId) -> ContractsChainSpec {
-	ContractsChainSpec::from_genesis(
-		"Contracts Local Testnet",
-		"contracts_local_testnet",
-		ChainType::Local,
-		move || {
-			contracts_testnet_genesis(
-				get_account_id_from_seed::<sr25519::Public>("Alice"),
-				vec![
-					get_account_id_from_seed::<sr25519::Public>("Alice"),
-					get_account_id_from_seed::<sr25519::Public>("Bob"),
-					get_account_id_from_seed::<sr25519::Public>("Charlie"),
-					get_account_id_from_seed::<sr25519::Public>("Dave"),
-					get_account_id_from_seed::<sr25519::Public>("Eve"),
-					get_account_id_from_seed::<sr25519::Public>("Ferdie"),
-					get_account_id_from_seed::<sr25519::Public>("Alice//stash"),
-					get_account_id_from_seed::<sr25519::Public>("Bob//stash"),
-					get_account_id_from_seed::<sr25519::Public>("Charlie//stash"),
-					get_account_id_from_seed::<sr25519::Public>("Dave//stash"),
-					get_account_id_from_seed::<sr25519::Public>("Eve//stash"),
-					get_account_id_from_seed::<sr25519::Public>("Ferdie//stash"),
-				],
-				id,
-			)
-		},
-		vec![],
-		None,
-		None,
-		None,
-		Extensions {
-			relay_chain: "westend-dev".into(),
-			para_id: id.into(),
-		},
-	)
-}
-
 pub fn staging_test_net(id: ParaId) -> ChainSpec {
 	ChainSpec::from_genesis(
 		"Staging Testnet",
@@ -184,28 +144,3 @@ fn testnet_genesis(
 		parachain_info: Some(parachain_runtime::ParachainInfoConfig { parachain_id: id }),
 	}
 }
-
-fn contracts_testnet_genesis(
-	root_key: AccountId,
-	endowed_accounts: Vec<AccountId>,
-	id: ParaId,
-) -> parachain_contracts_runtime::GenesisConfig {
-	parachain_contracts_runtime::GenesisConfig {
-		frame_system: Some(parachain_contracts_runtime::SystemConfig {
-			code: parachain_contracts_runtime::WASM_BINARY
-				.expect("WASM binary was not build, please build it!")
-				.to_vec(),
-			changes_trie_config: Default::default(),
-		}),
-		pallet_balances: Some(parachain_contracts_runtime::BalancesConfig {
-			balances: endowed_accounts
-				.iter()
-				.cloned()
-				.map(|k| (k, 1 << 60))
-				.collect(),
-		}),
-		pallet_sudo: Some(parachain_contracts_runtime::SudoConfig { key: root_key }),
-		parachain_info: Some(parachain_contracts_runtime::ParachainInfoConfig { parachain_id: id }),
-		cumulus_pallet_contracts: None,
-	}
-}

cumulus/rococo-parachains/src/cli.rs

@@ -22,9 +22,6 @@ use structopt::StructOpt;
 /// Sub-commands supported by the collator.
 #[derive(Debug, StructOpt)]
 pub enum Subcommand {
-	#[structopt(flatten)]
-	Base(sc_cli::Subcommand),
-
 	/// Export the genesis state of the parachain.
 	#[structopt(name = "export-genesis-state")]
 	ExportGenesisState(ExportGenesisStateCommand),
@@ -32,6 +29,27 @@ pub enum Subcommand {
 	/// Export the genesis wasm of the parachain.
 	#[structopt(name = "export-genesis-wasm")]
 	ExportGenesisWasm(ExportGenesisWasmCommand),
+
+	/// Build a chain specification.
+	BuildSpec(sc_cli::BuildSpecCmd),
+
+	/// Validate blocks.
+	CheckBlock(sc_cli::CheckBlockCmd),
+
+	/// Export blocks.
+	ExportBlocks(sc_cli::ExportBlocksCmd),
+
+	/// Export the state of a given block into a chain spec.
+	ExportState(sc_cli::ExportStateCmd),
+
+	/// Import blocks.
+	ImportBlocks(sc_cli::ImportBlocksCmd),
+
+	/// Remove the whole chain.
+	PurgeChain(sc_cli::PurgeChainCmd),
+
+	/// Revert the chain to a previous state.
+	Revert(sc_cli::RevertCmd),
 }
 
 /// Command for exporting the genesis state of the parachain

cumulus/rococo-parachains/src/command.rs

@@ -27,10 +27,13 @@ use sc_cli::{
 	ChainSpec, CliConfiguration, DefaultConfigurationValues, ImportParams, KeystoreParams,
 	NetworkParams, Result, RuntimeVersion, SharedParams, SubstrateCli,
 };
-use sc_service::config::{BasePath, PrometheusConfig};
+use sc_service::{
+	config::{BasePath, PrometheusConfig},
+	PartialComponents,
+};
 use sp_core::hexdisplay::HexDisplay;
 use sp_runtime::traits::Block as BlockT;
-use std::{io::Write, net::SocketAddr, sync::Arc};
+use std::{io::Write, net::SocketAddr};
 
 fn load_spec(
 	id: &str,
@@ -47,7 +50,6 @@ fn load_spec(
 		"track" => Ok(Box::new(chain_spec::ChainSpec::from_json_bytes(
 			&include_bytes!("../res/track.json")[..],
 		)?)),
-		"contracts" => Ok(Box::new(chain_spec::get_contracts_chain_spec(para_id))),
 		"" => Ok(Box::new(chain_spec::get_chain_spec(para_id))),
 		path => Ok(Box::new(chain_spec::ChainSpec::from_json_file(
 			path.into(),
@@ -143,50 +145,79 @@ fn extract_genesis_wasm(chain_spec: &Box<dyn sc_service::ChainSpec>) -> Result<V
 		.ok_or_else(|| "Could not find wasm file in genesis state!".into())
 }
 
-fn use_contracts_runtime(chain_spec: &Box<dyn ChainSpec>) -> bool {
-	chain_spec.id().starts_with("trick") || chain_spec.id().starts_with("contracts")
-}
-
 /// Parse command line arguments into service configuration.
 pub fn run() -> Result<()> {
 	let cli = Cli::from_args();
 
 	match &cli.subcommand {
-		Some(Subcommand::Base(subcommand)) => {
-			let runner = cli.create_runner(subcommand)?;
-
-			if use_contracts_runtime(&runner.config().chain_spec) {
-				runner.run_subcommand(subcommand, |mut config| {
-					let params = crate::service::new_partial::<
-						parachain_contracts_runtime::RuntimeApi,
-						crate::service::ContractsRuntimeExecutor,
-					>(&mut config)?;
-
-					Ok((
-						params.client,
-						params.backend,
-						params.import_queue,
-						params.task_manager,
-					))
-				})
-			} else {
-				runner.run_subcommand(subcommand, |mut config| {
-					let params = crate::service::new_partial::<
-						parachain_runtime::RuntimeApi,
-						crate::service::RuntimeExecutor,
-					>(&mut config)?;
-
-					Ok((
-						params.client,
-						params.backend,
-						params.import_queue,
-						params.task_manager,
-					))
-				})
-			}
+		Some(Subcommand::BuildSpec(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.sync_run(|config| cmd.run(config.chain_spec, config.network))
+		}
+		Some(Subcommand::CheckBlock(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let PartialComponents {
+					client,
+					task_manager,
+					import_queue,
+					..
+				} = crate::service::new_partial(&config)?;
+				Ok((cmd.run(client, import_queue), task_manager))
+			})
+		}
+		Some(Subcommand::ExportBlocks(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let PartialComponents {
+					client,
+					task_manager,
+					..
+				} = crate::service::new_partial(&config)?;
+				Ok((cmd.run(client, config.database), task_manager))
+			})
+		}
+		Some(Subcommand::ExportState(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let PartialComponents {
+					client,
+					task_manager,
+					..
+				} = crate::service::new_partial(&config)?;
+				Ok((cmd.run(client, config.chain_spec), task_manager))
+			})
+		}
+		Some(Subcommand::ImportBlocks(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let PartialComponents {
+					client,
+					task_manager,
+					import_queue,
+					..
+				} = crate::service::new_partial(&config)?;
+				Ok((cmd.run(client, import_queue), task_manager))
+			})
+		}
+		Some(Subcommand::PurgeChain(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.sync_run(|config| cmd.run(config.database))
+		}
+		Some(Subcommand::Revert(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let PartialComponents {
+					client,
+					task_manager,
+					backend,
+					..
+				} = crate::service::new_partial(&config)?;
+				Ok((cmd.run(client, backend), task_manager))
+			})
 		}
 		Some(Subcommand::ExportGenesisState(params)) => {
-			sc_cli::init_logger("");
+			sc_cli::init_logger("", sc_tracing::TracingReceiver::Log, None)?;
 
 			let block: Block = generate_genesis_block(&load_spec(
 				&params.chain.clone().unwrap_or_default(),
@@ -203,7 +234,7 @@ pub fn run() -> Result<()> {
 			Ok(())
 		}
 		Some(Subcommand::ExportGenesisWasm(params)) => {
-			sc_cli::init_logger("");
+			sc_cli::init_logger("", sc_tracing::TracingReceiver::Log, None)?;
 
 			let wasm_file =
 				extract_genesis_wasm(&cli.load_spec(&params.chain.clone().unwrap_or_default())?)?;
@@ -219,9 +250,9 @@ pub fn run() -> Result<()> {
 		None => {
 			let runner = cli.create_runner(&*cli.run)?;
 
-			runner.run_node_until_exit(|config| {
+			runner.run_node_until_exit(|config| async move {
 				// TODO
-				let key = Arc::new(sp_core::Pair::generate().0);
+				let key = sp_core::Pair::generate().0;
 
 				let extension = chain_spec::Extensions::try_get(&config.chain_spec);
 				let relay_chain_id = extension.map(|e| e.relay_chain.clone());
@@ -253,29 +284,11 @@ pub fn run() -> Result<()> {
 				info!("Parachain id: {:?}", id);
 				info!("Parachain Account: {}", parachain_account);
 				info!("Parachain genesis state: {}", genesis_state);
-				info!(
-					"Is collating: {}",
-					if collator { "yes" } else { "no" }
-				);
+				info!("Is collating: {}", if collator { "yes" } else { "no" });
 
-				if use_contracts_runtime(&config.chain_spec) {
-					crate::service::start_contracts_node(
-						config,
-						key,
-						polkadot_config,
-						id,
-						collator,
-					)
-				} else {
-					crate::service::start_node(
-						config,
-						key,
-						polkadot_config,
-						id,
-						collator,
-					)
+				crate::service::start_node(config, key, polkadot_config, id, collator)
+					.await
 					.map(|r| r.0)
-				}
 			})
 		}
 	}

cumulus/rococo-parachains/src/service.rs

@@ -14,42 +14,34 @@
 // You should have received a copy of the GNU General Public License
 // along with Cumulus.  If not, see <http://www.gnu.org/licenses/>.
 
-use ansi_term::Color;
-use cumulus_network::DelayedBlockAnnounceValidator;
+use cumulus_network::build_block_announce_validator;
 use cumulus_service::{
 	prepare_node_config, start_collator, start_full_node, StartCollatorParams, StartFullNodeParams,
 };
+use parachain_runtime::RuntimeApi;
 use polkadot_primitives::v0::CollatorPair;
 use rococo_parachain_primitives::Block;
 use sc_executor::native_executor_instance;
 pub use sc_executor::NativeExecutor;
-use sc_informant::OutputFormat;
 use sc_service::{Configuration, PartialComponents, Role, TFullBackend, TFullClient, TaskManager};
-use sp_api::ConstructRuntimeApi;
+use sp_core::Pair;
 use sp_runtime::traits::BlakeTwo256;
 use sp_trie::PrefixedMemoryDB;
 use std::sync::Arc;
 
 // Native executor instance.
 native_executor_instance!(
-	pub RuntimeExecutor,
+	pub Executor,
 	parachain_runtime::api::dispatch,
 	parachain_runtime::native_version,
 );
 
-// Native executor instance for the contracts runtime.
-native_executor_instance!(
-	pub ContractsRuntimeExecutor,
-	parachain_contracts_runtime::api::dispatch,
-	parachain_contracts_runtime::native_version,
-);
-
 /// Starts a `ServiceBuilder` for a full service.
 ///
 /// Use this macro if you don't actually need the full service, but just the builder in order to
 /// be able to perform chain operations.
-pub fn new_partial<RuntimeApi, Executor>(
-	config: &mut Configuration,
+pub fn new_partial(
+	config: &Configuration,
 ) -> Result<
 	PartialComponents<
 		TFullClient<Block, RuntimeApi, Executor>,
@@ -60,27 +52,10 @@ pub fn new_partial<RuntimeApi, Executor>(
 		(),
 	>,
 	sc_service::Error,
->
-where
-	RuntimeApi: ConstructRuntimeApi<Block, TFullClient<Block, RuntimeApi, Executor>>
-		+ Send
-		+ Sync
-		+ 'static,
-	RuntimeApi::RuntimeApi: sp_transaction_pool::runtime_api::TaggedTransactionQueue<Block>
-		+ sp_api::Metadata<Block>
-		+ sp_session::SessionKeys<Block>
-		+ sp_api::ApiExt<
-			Block,
-			Error = sp_blockchain::Error,
-			StateBackend = sc_client_api::StateBackendFor<TFullBackend<Block>, Block>,
-		> + sp_offchain::OffchainWorkerApi<Block>
-		+ sp_block_builder::BlockBuilder<Block>,
-	sc_client_api::StateBackendFor<TFullBackend<Block>, Block>: sp_api::StateBackend<BlakeTwo256>,
-	Executor: sc_executor::NativeExecutionDispatch + 'static,
-{
+> {
 	let inherent_data_providers = sp_inherents::InherentDataProviders::new();
 
-	let (client, backend, keystore, task_manager) =
+	let (client, backend, keystore_container, task_manager) =
 		sc_service::new_full_parts::<Block, RuntimeApi, Executor>(&config)?;
 	let client = Arc::new(client);
 
@@ -105,7 +80,7 @@ where
 		backend,
 		client,
 		import_queue,
-		keystore,
+		keystore_container,
 		task_manager,
 		transaction_pool,
 		inherent_data_providers,
@@ -119,30 +94,15 @@ where
 /// Start a node with the given parachain `Configuration` and relay chain `Configuration`.
 ///
 /// This is the actual implementation that is abstract over the executor and the runtime api.
-fn start_node_impl<RuntimeApi, Executor, RB>(
+async fn start_node_impl<RB>(
 	parachain_config: Configuration,
-	collator_key: Arc<CollatorPair>,
-	mut polkadot_config: polkadot_collator::Configuration,
+	collator_key: CollatorPair,
+	polkadot_config: Configuration,
 	id: polkadot_primitives::v0::Id,
 	validator: bool,
 	rpc_ext_builder: RB,
 ) -> sc_service::error::Result<(TaskManager, Arc<TFullClient<Block, RuntimeApi, Executor>>)>
 where
-	RuntimeApi: ConstructRuntimeApi<Block, TFullClient<Block, RuntimeApi, Executor>>
-		+ Send
-		+ Sync
-		+ 'static,
-	RuntimeApi::RuntimeApi: sp_transaction_pool::runtime_api::TaggedTransactionQueue<Block>
-		+ sp_api::Metadata<Block>
-		+ sp_session::SessionKeys<Block>
-		+ sp_api::ApiExt<
-			Block,
-			Error = sp_blockchain::Error,
-			StateBackend = sc_client_api::StateBackendFor<TFullBackend<Block>, Block>,
-		> + sp_offchain::OffchainWorkerApi<Block>
-		+ sp_block_builder::BlockBuilder<Block>,
-	sc_client_api::StateBackendFor<TFullBackend<Block>, Block>: sp_api::StateBackend<BlakeTwo256>,
-	Executor: sc_executor::NativeExecutionDispatch + 'static,
 	RB: Fn(
 			Arc<TFullClient<Block, RuntimeApi, Executor>>,
 		) -> jsonrpc_core::IoHandler<sc_rpc::Metadata>
@@ -153,18 +113,12 @@ where
 		return Err("Light client not supported!".into());
 	}
 
-	let mut parachain_config = prepare_node_config(parachain_config);
+	let parachain_config = prepare_node_config(parachain_config);
 
-	parachain_config.informant_output_format = OutputFormat {
-		enable_color: true,
-		prefix: format!("[{}] ", Color::Yellow.bold().paint("Parachain")),
-	};
-	polkadot_config.informant_output_format = OutputFormat {
-		enable_color: true,
-		prefix: format!("[{}] ", Color::Blue.bold().paint("Relaychain")),
-	};
+	let polkadot_full_node =
+		cumulus_service::build_polkadot_full_node(polkadot_config, collator_key.public())?;
 
-	let params = new_partial::<RuntimeApi, Executor>(&mut parachain_config)?;
+	let params = new_partial(&parachain_config)?;
 	params
 		.inherent_data_providers
 		.register_provider(sp_timestamp::InherentDataProvider)
@@ -172,11 +126,11 @@ where
 
 	let client = params.client.clone();
 	let backend = params.backend.clone();
-	let block_announce_validator = DelayedBlockAnnounceValidator::new();
-	let block_announce_validator_builder = {
-		let block_announce_validator = block_announce_validator.clone();
-		move |_| Box::new(block_announce_validator) as Box<_>
-	};
+	let block_announce_validator = build_block_announce_validator(
+		polkadot_full_node.client.clone(),
+		id,
+		Box::new(polkadot_full_node.network.clone()),
+	);
 
 	let prometheus_registry = parachain_config.prometheus_registry().cloned();
 	let transaction_pool = params.transaction_pool.clone();
@@ -190,16 +144,13 @@ where
 			spawn_handle: task_manager.spawn_handle(),
 			import_queue,
 			on_demand: None,
-			block_announce_validator_builder: Some(Box::new(block_announce_validator_builder)),
+			block_announce_validator_builder: Some(Box::new(|_| block_announce_validator)),
 			finality_proof_request_builder: None,
 			finality_proof_provider: None,
 		})?;
 
-	let rpc_extensions_builder = {
-		let client = client.clone();
-
-		Box::new(move |_deny_unsafe| rpc_ext_builder(client.clone()))
-	};
+	let rpc_client = client.clone();
+	let rpc_extensions_builder = Box::new(move |_, _| rpc_ext_builder(rpc_client.clone()));
 
 	sc_service::spawn_tasks(sc_service::SpawnTasksParams {
 		on_demand: None,
@@ -210,8 +161,8 @@ where
 		task_manager: &mut task_manager,
 		telemetry_connection_sinks: Default::default(),
 		config: parachain_config,
-		keystore: params.keystore,
-		backend,
+		keystore: params.keystore_container.sync_keystore(),
+		backend: backend.clone(),
 		network: network.clone(),
 		network_status_sinks,
 		system_rpc_tx,
@@ -224,10 +175,12 @@ where
 
 	if validator {
 		let proposer_factory = sc_basic_authorship::ProposerFactory::new(
+			task_manager.spawn_handle(),
 			client.clone(),
 			transaction_pool,
 			prometheus_registry.as_ref(),
 		);
+		let spawner = task_manager.spawn_handle();
 
 		let params = StartCollatorParams {
 			para_id: id,
@@ -237,22 +190,21 @@ where
 			block_status: client.clone(),
 			announce_block,
 			client: client.clone(),
-			block_announce_validator,
 			task_manager: &mut task_manager,
-			polkadot_config,
 			collator_key,
+			polkadot_full_node,
+			spawner,
+			backend,
 		};
 
-		start_collator(params)?;
+		start_collator(params).await?;
 	} else {
 		let params = StartFullNodeParams {
 			client: client.clone(),
 			announce_block,
-			polkadot_config,
-			collator_key,
-			block_announce_validator,
 			task_manager: &mut task_manager,
 			para_id: id,
+			polkadot_full_node,
 		};
 
 		start_full_node(params)?;
@@ -264,17 +216,14 @@ where
 }
 
 /// Start a normal parachain node.
-pub fn start_node(
+pub async fn start_node(
 	parachain_config: Configuration,
-	collator_key: Arc<CollatorPair>,
-	polkadot_config: polkadot_collator::Configuration,
+	collator_key: CollatorPair,
+	polkadot_config: Configuration,
 	id: polkadot_primitives::v0::Id,
 	validator: bool,
-) -> sc_service::error::Result<(
-	TaskManager,
-	Arc<TFullClient<Block, parachain_runtime::RuntimeApi, RuntimeExecutor>>,
-)> {
-	start_node_impl::<parachain_runtime::RuntimeApi, RuntimeExecutor, _>(
+) -> sc_service::error::Result<(TaskManager, Arc<TFullClient<Block, RuntimeApi, Executor>>)> {
+	start_node_impl(
 		parachain_config,
 		collator_key,
 		polkadot_config,
@@ -282,29 +231,5 @@ pub fn start_node(
 		validator,
 		|_| Default::default(),
 	)
-}
-
-/// Start a contracts parachain node.
-pub fn start_contracts_node(
-	parachain_config: Configuration,
-	collator_key: Arc<CollatorPair>,
-	polkadot_config: polkadot_collator::Configuration,
-	id: polkadot_primitives::v0::Id,
-	validator: bool,
-) -> sc_service::error::Result<TaskManager> {
-	start_node_impl::<parachain_contracts_runtime::RuntimeApi, ContractsRuntimeExecutor, _>(
-		parachain_config,
-		collator_key,
-		polkadot_config,
-		id,
-		validator,
-		|client| {
-			let mut io = jsonrpc_core::IoHandler::default();
-
-			use cumulus_pallet_contracts_rpc::{Contracts, ContractsApi};
-			io.extend_with(ContractsApi::to_delegate(Contracts::new(client)));
-			io
-		},
-	)
-	.map(|r| r.0)
+	.await
 }