Update polkadot & substrate (#76)

cumulus/test/parachain/Cargo.toml

@@ -50,7 +50,7 @@ polkadot-service = { git = "https://github.com/paritytech/polkadot", branch = "c
 polkadot-cli = { git = "https://github.com/paritytech/polkadot", branch = "cumulus-branch" }
 
 [build-dependencies]
-vergen = '3.0.4'
+substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "cumulus-branch" }
 
 [dev-dependencies]
 assert_cmd = "0.12"

cumulus/test/parachain/build.rs

@@ -16,12 +16,11 @@
 
 use std::{env, path::PathBuf};
 
-use vergen::{generate_cargo_keys, ConstantsFlags};
-
-const ERROR_MSG: &str = "Failed to generate metadata files";
+use substrate_build_script_utils::{generate_cargo_keys, rerun_if_git_head_changed};
 
 fn main() {
-	generate_cargo_keys(ConstantsFlags::SHA_SHORT).expect(ERROR_MSG);
+	generate_cargo_keys();
+	rerun_if_git_head_changed();
 
 	let mut manifest_dir = PathBuf::from(
 		env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is always set by cargo."),

cumulus/test/parachain/src/chain_spec.rs

@@ -43,6 +43,7 @@ pub fn get_chain_spec() -> ChainSpec {
 	ChainSpec::from_genesis(
 		"Local Testnet",
 		"parachain_local_testnet",
+		sc_service::ChainType::Local,
 		|| {
 			testnet_genesis(
 				vec![

cumulus/test/parachain/src/cli.rs

@@ -17,7 +17,6 @@
 use std::path::PathBuf;
 
 use sc_cli;
-pub use polkadot_cli::Cli as PolkadotCli;
 use structopt::StructOpt;
 
 /// Sub-commands supported by the collator.
@@ -55,3 +54,12 @@ pub struct Cli {
 	#[structopt(raw = true)]
 	pub relaychain_args: Vec<String>,
 }
+
+#[derive(Debug, StructOpt, Clone)]
+pub struct PolkadotCli {
+	#[structopt(flatten)]
+	pub base: polkadot_cli::RunCmd,
+
+	#[structopt(skip)]
+	pub base_path: Option<PathBuf>,
+}

cumulus/test/parachain/src/command.rs

@@ -16,52 +16,125 @@
 
 use crate::chain_spec;
 use crate::cli::{Cli, PolkadotCli, Subcommand};
-
-use std::sync::Arc;
-
+use codec::Encode;
+use log::info;
 use parachain_runtime::Block;
-
+use sc_cli::{
+	CliConfiguration, Error, ImportParams, KeystoreParams, NetworkParams, Result, SharedParams,
+	SubstrateCli,
+};
 use sc_client::genesis;
-use sc_service::{Configuration, Role as ServiceRole, config::PrometheusConfig};
+use sc_network::config::TransportConfig;
+use sc_service::{
+	config::{NetworkConfiguration, NodeKeyConfig, PrometheusConfig},
+	Configuration, Role as ServiceRole,
+};
 use sp_core::hexdisplay::HexDisplay;
 use sp_runtime::{
 	traits::{Block as BlockT, Hash as HashT, Header as HeaderT},
 	BuildStorage,
 };
-use sc_network::config::TransportConfig;
+use std::net::SocketAddr;
+use std::path::PathBuf;
+use std::sync::Arc;
 
-use codec::Encode;
-use log::info;
+impl SubstrateCli for Cli {
+	fn impl_name() -> &'static str {
+		"Cumulus Test Parachain Collator"
+	}
 
-const DEFAULT_POLKADOT_RPC_HTTP: &'static str = "127.0.0.1:9934";
-const DEFAULT_POLKADOT_RPC_WS: &'static str = "127.0.0.1:9945";
-const DEFAULT_POLKADOT_PROMETHEUS_PORT: &'static str = "127.0.0.1:9616";
+	fn impl_version() -> &'static str {
+		env!("SUBSTRATE_CLI_IMPL_VERSION")
+	}
+
+	fn description() -> &'static str {
+		"Cumulus test parachain collator\n\nThe command-line arguments provided first will be \
+		passed to the parachain node, while the arguments provided after -- will be passed \
+		to the relaychain node.\n\n\
+		cumulus-test-parachain-collator [parachain-args] -- [relaychain-args]"
+	}
+
+	fn author() -> &'static str {
+		env!("CARGO_PKG_AUTHORS")
+	}
+
+	fn support_url() -> &'static str {
+		"https://github.com/paritytech/cumulus/issues/new"
+	}
+
+	fn copyright_start_year() -> i32 {
+		2017
+	}
+
+	fn executable_name() -> &'static str {
+		"cumulus-test-parachain-collator"
+	}
+
+	fn load_spec(&self, _id: &str) -> std::result::Result<Box<dyn sc_service::ChainSpec>, String> {
+		Ok(Box::new(chain_spec::get_chain_spec()))
+	}
+}
+
+impl SubstrateCli for PolkadotCli {
+	fn impl_name() -> &'static str {
+		"Cumulus Test Parachain Collator"
+	}
+
+	fn impl_version() -> &'static str {
+		env!("SUBSTRATE_CLI_IMPL_VERSION")
+	}
+
+	fn description() -> &'static str {
+		"Cumulus test parachain collator\n\nThe command-line arguments provided first will be \
+		passed to the parachain node, while the arguments provided after -- will be passed \
+		to the relaychain node.\n\n\
+		cumulus-test-parachain-collator [parachain-args] -- [relaychain-args]"
+	}
+
+	fn author() -> &'static str {
+		env!("CARGO_PKG_AUTHORS")
+	}
+
+	fn support_url() -> &'static str {
+		"https://github.com/paritytech/cumulus/issues/new"
+	}
+
+	fn copyright_start_year() -> i32 {
+		2017
+	}
+
+	fn executable_name() -> &'static str {
+		"cumulus-test-parachain-collator"
+	}
+
+	fn load_spec(&self, _id: &str) -> std::result::Result<Box<dyn sc_service::ChainSpec>, String> {
+		polkadot_service::PolkadotChainSpec::from_json_bytes(
+			&include_bytes!("../res/polkadot_chainspec.json")[..],
+		)
+		.map(|r| Box::new(r) as Box<_>)
+	}
+}
 
 /// Parse command line arguments into service configuration.
-pub fn run(version: sc_cli::VersionInfo) -> sc_cli::Result<()> {
-	let opt: Cli = sc_cli::from_args(&version);
+pub fn run() -> Result<()> {
+	let cli = Cli::from_args();
 
-	let mut config = sc_service::Configuration::from_version(&version);
-	let mut polkadot_config = Configuration::from_version(&version);
-
-	match opt.subcommand {
+	match &cli.subcommand {
 		Some(Subcommand::Base(subcommand)) => {
-			subcommand.init(&version)?;
-			subcommand.update_config(&mut config, load_spec, &version)?;
-			subcommand.run(
-				config,
-				|config: Configuration| Ok(new_full_start!(config).0),
-			)
-		},
+			let runner = cli.create_runner(subcommand)?;
+
+			runner.run_subcommand(subcommand, |config| Ok(new_full_start!(config).0))
+		}
 		Some(Subcommand::ExportGenesisState(params)) => {
 			sc_cli::init_logger("");
 
 			let storage = (&chain_spec::get_chain_spec()).build_storage()?;
 
 			let child_roots = storage.children.iter().map(|(sk, child_content)| {
-				let state_root = <<<Block as BlockT>::Header as HeaderT>::Hashing as HashT>::trie_root(
-					child_content.data.clone().into_iter().collect(),
-				);
+				let state_root =
+					<<<Block as BlockT>::Header as HeaderT>::Hashing as HashT>::trie_root(
+						child_content.data.clone().into_iter().collect(),
+					);
 				(sk.clone(), state_root.encode())
 			});
 			let state_root = <<<Block as BlockT>::Header as HeaderT>::Hashing as HashT>::trie_root(
@@ -71,74 +144,211 @@ pub fn run(version: sc_cli::VersionInfo) -> sc_cli::Result<()> {
 
 			let header_hex = format!("0x{:?}", HexDisplay::from(&block.header().encode()));
 
-			if let Some(output) = params.output {
+			if let Some(output) = &params.output {
 				std::fs::write(output, header_hex)?;
 			} else {
 				println!("{}", header_hex);
 			}
 
 			Ok(())
-		},
+		}
 		None => {
-			opt.run.init(&version)?;
-			opt.run.update_config(&mut config, load_spec, &version)?;
-
-			info!("{}", version.name);
-			info!("  version {}", config.full_version());
-			info!("  by {}, 2019", version.author);
-			info!("Chain specification: {}", config.expect_chain_spec().name());
-			info!("Node name: {}", config.name);
-			info!("Roles: {:?}", config.role);
-			info!("Parachain id: {:?}", crate::PARA_ID);
+			let runner = cli.create_runner(&cli.run)?;
 
 			// TODO
 			let key = Arc::new(sp_core::Pair::from_seed(&[10; 32]));
+			let key2 = Arc::new(sp_core::Pair::from_seed(&[10; 32]));
 
-			polkadot_config.config_dir = config.in_chain_config_dir("polkadot");
-
-			let polkadot_opt: PolkadotCli = sc_cli::from_iter(
-				[version.executable_name.to_string()].iter().chain(opt.relaychain_args.iter()),
-				&version,
-			);
-			let allow_private_ipv4 = !polkadot_opt.run.base.network_config.no_private_ipv4;
-
-			polkadot_config.rpc_http = Some(DEFAULT_POLKADOT_RPC_HTTP.parse().unwrap());
-			polkadot_config.rpc_ws = Some(DEFAULT_POLKADOT_RPC_WS.parse().unwrap());
-			polkadot_config.prometheus_config = Some(
-				PrometheusConfig::new_with_default_registry(
-					DEFAULT_POLKADOT_PROMETHEUS_PORT.parse().unwrap(),
-				)
+			let mut polkadot_cli = PolkadotCli::from_iter(
+				[PolkadotCli::executable_name().to_string()]
+					.iter()
+					.chain(cli.relaychain_args.iter()),
 			);
 
-			polkadot_opt.run.base.update_config(
-				&mut polkadot_config,
-				load_spec_polkadot,
-				&version,
-			)?;
+			polkadot_cli.base_path = cli.run.base_path()?.map(|x| x.join("polkadot"));
 
-			// TODO: we disable mdns for the polkadot node because it prevents the process to exit
-			//       properly. See https://github.com/paritytech/cumulus/issues/57
-			polkadot_config.network.transport = TransportConfig::Normal {
-				enable_mdns: false,
-				allow_private_ipv4,
-				wasm_external_transport: None,
-				use_yamux_flow_control: false,
-			};
+			runner.run_node(
+				|config| {
+					let task_executor = config.task_executor.clone();
+					let polkadot_config = SubstrateCli::create_configuration(
+						&polkadot_cli,
+						&polkadot_cli,
+						task_executor,
+					)
+					.unwrap();
 
-			match config.role {
-				ServiceRole::Light => unimplemented!("Light client not supported!"),
-				_ => crate::service::run_collator(config, key, polkadot_config),
-			}
-		},
+					info!("Parachain id: {:?}", crate::PARA_ID);
+
+					crate::service::run_collator(config, key2, polkadot_config)
+				},
+				|config| {
+					let task_executor = config.task_executor.clone();
+					let polkadot_config = SubstrateCli::create_configuration(
+						&polkadot_cli,
+						&polkadot_cli,
+						task_executor,
+					)
+					.unwrap();
+
+					info!("Parachain id: {:?}", crate::PARA_ID);
+
+					crate::service::run_collator(config, key, polkadot_config)
+				},
+			)
+		}
 	}
 }
 
-fn load_spec(_: &str) -> Result<Box<dyn sc_service::ChainSpec>, String> {
-	Ok(Box::new(chain_spec::get_chain_spec()))
+impl CliConfiguration for PolkadotCli {
+	fn shared_params(&self) -> &SharedParams {
+		self.base.base.shared_params()
+	}
+
+	fn import_params(&self) -> Option<&ImportParams> {
+		self.base.base.import_params()
+	}
+
+	fn network_params(&self) -> Option<&NetworkParams> {
+		self.base.base.network_params()
+	}
+
+	fn keystore_params(&self) -> Option<&KeystoreParams> {
+		self.base.base.keystore_params()
+	}
+
+	fn base_path(&self) -> Result<Option<PathBuf>> {
+		Ok(self.shared_params().base_path().or(self.base_path.clone()))
+	}
+
+	fn rpc_http(&self) -> Result<Option<SocketAddr>> {
+		let rpc_external = self.base.base.rpc_external;
+		let unsafe_rpc_external = self.base.base.unsafe_rpc_external;
+		let validator = self.base.base.validator;
+		let rpc_port = self.base.base.rpc_port;
+		// copied directly from substrate
+		let rpc_interface: &str = interface_str(rpc_external, unsafe_rpc_external, validator)?;
+
+		Ok(Some(parse_address(
+			&format!("{}:{}", rpc_interface, 9934),
+			rpc_port,
+		)?))
+	}
+
+	fn rpc_ws(&self) -> Result<Option<SocketAddr>> {
+		let ws_external = self.base.base.ws_external;
+		let unsafe_ws_external = self.base.base.unsafe_ws_external;
+		let validator = self.base.base.validator;
+		let ws_port = self.base.base.ws_port;
+		// copied directly from substrate
+		let ws_interface: &str = interface_str(ws_external, unsafe_ws_external, validator)?;
+
+		Ok(Some(parse_address(
+			&format!("{}:{}", ws_interface, 9945),
+			ws_port,
+		)?))
+	}
+
+	fn prometheus_config(&self) -> Result<Option<PrometheusConfig>> {
+		let no_prometheus = self.base.base.no_prometheus;
+		let prometheus_external = self.base.base.prometheus_external;
+		let prometheus_port = self.base.base.prometheus_port;
+
+		if no_prometheus {
+			Ok(None)
+		} else {
+			let prometheus_interface: &str = if prometheus_external {
+				"0.0.0.0"
+			} else {
+				"127.0.0.1"
+			};
+
+			Ok(Some(PrometheusConfig::new_with_default_registry(
+				parse_address(
+					&format!("{}:{}", prometheus_interface, 9616),
+					prometheus_port,
+				)?,
+			)))
+		}
+	}
+
+	// TODO: we disable mdns for the polkadot node because it prevents the process to exit
+	//       properly. See https://github.com/paritytech/cumulus/issues/57
+	fn network_config(
+		&self,
+		chain_spec: &Box<dyn sc_service::ChainSpec>,
+		is_dev: bool,
+		net_config_dir: &PathBuf,
+		client_id: &str,
+		node_name: &str,
+		node_key: NodeKeyConfig,
+	) -> Result<NetworkConfiguration> {
+		let (mut network, allow_private_ipv4) = self
+			.network_params()
+			.map(|x| {
+				(
+					x.network_config(
+						chain_spec,
+						is_dev,
+						net_config_dir,
+						client_id,
+						node_name,
+						node_key,
+					),
+					!x.no_private_ipv4,
+				)
+			})
+			.expect("NetworkParams is always available on RunCmd; qed");
+
+		network.transport = TransportConfig::Normal {
+			enable_mdns: false,
+			allow_private_ipv4,
+			wasm_external_transport: None,
+			use_yamux_flow_control: false,
+		};
+
+		Ok(network)
+	}
+
+	fn init<C: SubstrateCli>(&self) -> Result<()> {
+		unreachable!("PolkadotCli is never initialized; qed");
+	}
 }
 
-fn load_spec_polkadot(_: &str) -> Result<Box<dyn sc_service::ChainSpec>, String> {
-	polkadot_service::PolkadotChainSpec::from_json_bytes(
-		&include_bytes!("../res/polkadot_chainspec.json")[..],
-	).map(|r| Box::new(r) as Box<_>)
+// copied directly from substrate
+fn parse_address(address: &str, port: Option<u16>) -> std::result::Result<SocketAddr, String> {
+	let mut address: SocketAddr = address
+		.parse()
+		.map_err(|_| format!("Invalid address: {}", address))?;
+	if let Some(port) = port {
+		address.set_port(port);
+	}
+
+	Ok(address)
+}
+
+// copied directly from substrate
+fn interface_str(
+	is_external: bool,
+	is_unsafe_external: bool,
+	is_validator: bool,
+) -> Result<&'static str> {
+	if is_external && is_validator {
+		return Err(Error::Input(
+			"--rpc-external and --ws-external options shouldn't be \
+		used if the node is running as a validator. Use `--unsafe-rpc-external` if you understand \
+		the risks. See the options description for more information."
+				.to_owned(),
+		));
+	}
+
+	if is_external || is_unsafe_external {
+		log::warn!(
+			"It isn't safe to expose RPC publicly without a proxy server that filters \
+		available set of RPC methods."
+		);
+
+		Ok("0.0.0.0")
+	} else {
+		Ok("127.0.0.1")
+	}
 }

cumulus/test/parachain/src/main.rs

@@ -29,24 +29,7 @@ mod command;
 
 /// The parachain id of this parachain.
 pub const PARA_ID: ParaId = ParaId::new(100);
-const EXECUTABLE_NAME: &'static str = "cumulus-test-parachain-collator";
-const DESCRIPTION: &'static str =
-	"Cumulus test parachain collator\n\nThe command-line arguments provided first will be \
-	passed to the parachain node, while the arguments provided after -- will be passed \
-	to the relaychain node.\n\n\
-	cumulus-test-parachain-collator [parachain-args] -- [relaychain-args]";
 
 fn main() -> sc_cli::Result<()> {
-	let version = sc_cli::VersionInfo {
-		name: "Cumulus Test Parachain Collator",
-		commit: env!("VERGEN_SHA_SHORT"),
-		version: env!("CARGO_PKG_VERSION"),
-		author: "Parity Technologies <admin@parity.io>",
-		description: DESCRIPTION,
-		executable_name: EXECUTABLE_NAME,
-		support_url: "https://github.com/paritytech/cumulus/issues/new",
-		copyright_start_year: 2017,
-	};
-
-	command::run(version)
+	command::run()
 }

cumulus/test/parachain/src/service.rs

@@ -74,47 +74,43 @@ macro_rules! new_full_start {
 pub fn run_collator(
 	parachain_config: Configuration,
 	key: Arc<CollatorPair>,
-	mut polkadot_config: polkadot_collator::Configuration,
-) -> sc_cli::Result<()> {
-	sc_cli::run_service_until_exit(parachain_config, move |parachain_config| {
-		polkadot_config.task_executor = parachain_config.task_executor.clone();
+	polkadot_config: polkadot_collator::Configuration,
+) -> sc_service::error::Result<impl AbstractService> {
+	let (builder, inherent_data_providers) = new_full_start!(parachain_config);
+	inherent_data_providers
+		.register_provider(sp_timestamp::InherentDataProvider)
+		.unwrap();
 
-		let (builder, inherent_data_providers) = new_full_start!(parachain_config);
-		inherent_data_providers
-			.register_provider(sp_timestamp::InherentDataProvider)
-			.unwrap();
+	let service = builder
+		.with_finality_proof_provider(|client, backend| {
+			// GenesisAuthoritySetProvider is implemented for StorageAndProofProvider
+			let provider = client as Arc<dyn StorageAndProofProvider<_, _>>;
+			Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, provider)) as _)
+		})?
+		.build()?;
 
-		let service = builder
-			.with_finality_proof_provider(|client, backend| {
-				// GenesisAuthoritySetProvider is implemented for StorageAndProofProvider
-				let provider = client as Arc<dyn StorageAndProofProvider<_, _>>;
-				Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, provider)) as _)
-			})?
-			.build()?;
+	let proposer_factory = sc_basic_authorship::ProposerFactory::new(
+		service.client(),
+		service.transaction_pool(),
+	);
 
-		let proposer_factory = sc_basic_authorship::ProposerFactory::new(
-			service.client(),
-			service.transaction_pool(),
-		);
+	let block_import = service.client();
+	let client = service.client();
+	let builder = CollatorBuilder::new(
+		proposer_factory,
+		inherent_data_providers,
+		block_import,
+		crate::PARA_ID,
+		client,
+	);
 
-		let block_import = service.client();
-		let client = service.client();
-		let builder = CollatorBuilder::new(
-			proposer_factory,
-			inherent_data_providers,
-			block_import,
-			crate::PARA_ID,
-			client,
-		);
+	let polkadot_future = polkadot_collator::start_collator(
+		builder,
+		crate::PARA_ID,
+		key,
+		polkadot_config,
+	).map(|_| ());
+	service.spawn_essential_task("polkadot", polkadot_future);
 
-		let polkadot_future = polkadot_collator::start_collator(
-			builder,
-			crate::PARA_ID,
-			key,
-			polkadot_config,
-		).map(|_| ());
-		service.spawn_essential_task("polkadot", polkadot_future);
-
-		Ok(service)
-	})
+	Ok(service)
 }

cumulus/test/parachain/tests/polkadot_argument_parsing.rs

@@ -32,9 +32,11 @@ fn polkadot_argument_parsing() {
 		let _ = fs::remove_dir_all("polkadot_argument_parsing");
 		let mut cmd = Command::new(cargo_bin("cumulus-test-parachain-collator"))
 			.args(&[
+				"--dev",
 				"-d",
 				"polkadot_argument_parsing",
 				"--",
+				"--dev",
 				"--bootnodes",
 				"/ip4/127.0.0.1/tcp/30333/p2p/Qmbx43psh7LVkrYTRXisUpzCubbgYojkejzAgj5mteDnxy",
 				"--bootnodes",