Update polkadot & substrate (#76)

cumulus/.editorconfig

@@ -8,3 +8,15 @@ charset=utf-8
 trim_trailing_whitespace=true
 max_line_length=100
 insert_final_newline=true
+
+[*.yml]
+indent_style=space
+indent_size=2
+tab_width=8
+end_of_line=lf
+
+[*.sh]
+indent_style=space
+indent_size=4
+tab_width=8
+end_of_line=lf

cumulus/collator/src/lib.rs

@@ -31,8 +31,8 @@ use polkadot_collator::{
 	PolkadotClient,
 };
 use polkadot_primitives::{
-	parachain::{self, BlockData, LocalValidationData, Id as ParaId}, Block as PBlock,
+	parachain::{self, BlockData, GlobalValidationSchedule, LocalValidationData, Id as ParaId},
-	Hash as PHash,
+	Block as PBlock, Hash as PHash,
 };
 
 use codec::{Decode, Encode};
@@ -111,7 +111,8 @@ where
 	fn produce_candidate(
 		&mut self,
 		_relay_chain_parent: PHash,
-		status: LocalValidationData,
+		_global_validation: GlobalValidationSchedule,
+		local_validation: LocalValidationData,
 	) -> Self::ProduceCandidate {
 		let factory = self.proposer_factory.clone();
 		let inherent_providers = self.inherent_data_providers.clone();
@@ -119,7 +120,7 @@ where
 
 		trace!(target: "cumulus-collator", "Producing candidate");
 
-		let last_head = match HeadData::<Block>::decode(&mut &status.parent_head.0[..]) {
+		let last_head = match HeadData::<Block>::decode(&mut &local_validation.parent_head.0[..]) {
 			Ok(x) => x,
 			Err(e) => {
 				error!(target: "cumulus-collator", "Could not decode the head data: {:?}", e);
@@ -479,9 +480,15 @@ mod tests {
 		let collation = collate(
 			Default::default(),
 			id,
+			GlobalValidationSchedule {
+				block_number: 0,
+				max_code_size: 0,
+				max_head_data_size: 0,
+			},
 			LocalValidationData {
 				parent_head: HeadData(header.encode()),
 				balance: 10,
+				code_upgrade_allowed: None,
 			},
 			context,
 			Arc::new(Sr25519Keyring::Alice.pair().into()),

cumulus/runtime/src/validate_block/implementation.rs

@@ -28,7 +28,7 @@ use hash_db::{HashDB, EMPTY_PREFIX};
 
 use trie_db::{Trie, TrieDB};
 
-use parachain::{ValidationParams, ValidationResult};
+use parachain::primitives::{HeadData, ValidationParams, ValidationResult};
 
 use codec::{Decode, Encode};
 
@@ -73,12 +73,12 @@ trait Storage {
 /// Validate a given parachain block on a validator.
 #[doc(hidden)]
 pub fn validate_block<B: BlockT, E: ExecuteBlock<B>>(params: ValidationParams) -> ValidationResult {
-	let block_data = crate::ParachainBlockData::<B>::decode(&mut &params.block_data[..])
+	let block_data = crate::ParachainBlockData::<B>::decode(&mut &params.block_data.0[..])
 		.expect("Invalid parachain block data");
 
-	let parent_head = B::Header::decode(&mut &params.parent_head[..]).expect("Invalid parent head");
+	let parent_head = B::Header::decode(&mut &params.parent_head.0[..]).expect("Invalid parent head");
 	// TODO: Use correct head data
-	let head_data = block_data.header.encode();
+	let head_data = HeadData(block_data.header.encode());
 
 	// TODO: Add `PolkadotInherent`.
 	let block = B::new(block_data.header, block_data.extrinsics);
@@ -110,7 +110,10 @@ pub fn validate_block<B: BlockT, E: ExecuteBlock<B>>(params: ValidationParams) -
 
 	E::execute_block(block);
 
-	ValidationResult { head_data }
+	ValidationResult {
+		head_data,
+		new_validation_code: None,
+	}
 }
 
 /// The storage implementation used when validating a block that is using the

cumulus/runtime/src/validate_block/tests.rs

@@ -16,7 +16,7 @@
 
 use crate::{ParachainBlockData, WitnessData};
 
-use parachain::{ValidationParams, ValidationResult};
+use parachain::primitives::{BlockData, HeadData, ValidationParams, ValidationResult};
 use sc_executor::{
 	error::Result, WasmExecutionMethod, WasmExecutor, sp_wasm_interface::HostFunctions,
 };
@@ -44,8 +44,12 @@ fn call_validate_block(
 	let mut ext = TestExternalities::default();
 	let mut ext_ext = ext.ext();
 	let params = ValidationParams {
-		block_data: block_data.encode(),
+		block_data: BlockData(block_data.encode()),
-		parent_head: parent_head.encode(),
+		parent_head: HeadData(parent_head.encode()),
+		code_upgrade_allowed: None,
+		max_code_size: 1024,
+		max_head_data_size: 1024,
+		relay_chain_height: 1,
 	}
 	.encode();
 
@@ -65,7 +69,7 @@ fn call_validate_block(
 		&mut ext_ext,
 	)
 	.map(|v| ValidationResult::decode(&mut &v[..]).expect("Decode `ValidationResult`."))
-	.map(|v| Header::decode(&mut &v.head_data[..]).expect("Decode `Header`."))
+	.map(|v| Header::decode(&mut &v.head_data.0[..]).expect("Decode `Header`."))
 	.map_err(|err| err.into())
 }
 

cumulus/test/parachain/Cargo.toml

@@ -50,7 +50,7 @@ polkadot-service = { git = "https://github.com/paritytech/polkadot", branch = "c
 polkadot-cli = { git = "https://github.com/paritytech/polkadot", branch = "cumulus-branch" }
 
 [build-dependencies]
-vergen = '3.0.4'
+substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "cumulus-branch" }
 
 [dev-dependencies]
 assert_cmd = "0.12"

cumulus/test/parachain/build.rs

@@ -16,12 +16,11 @@
 
 use std::{env, path::PathBuf};
 
-use vergen::{generate_cargo_keys, ConstantsFlags};
+use substrate_build_script_utils::{generate_cargo_keys, rerun_if_git_head_changed};
-
-const ERROR_MSG: &str = "Failed to generate metadata files";
 
 fn main() {
-	generate_cargo_keys(ConstantsFlags::SHA_SHORT).expect(ERROR_MSG);
+	generate_cargo_keys();
+	rerun_if_git_head_changed();
 
 	let mut manifest_dir = PathBuf::from(
 		env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is always set by cargo."),

cumulus/test/parachain/src/chain_spec.rs

@@ -43,6 +43,7 @@ pub fn get_chain_spec() -> ChainSpec {
 	ChainSpec::from_genesis(
 		"Local Testnet",
 		"parachain_local_testnet",
+		sc_service::ChainType::Local,
 		|| {
 			testnet_genesis(
 				vec![

cumulus/test/parachain/src/cli.rs

@@ -17,7 +17,6 @@
 use std::path::PathBuf;
 
 use sc_cli;
-pub use polkadot_cli::Cli as PolkadotCli;
 use structopt::StructOpt;
 
 /// Sub-commands supported by the collator.
@@ -55,3 +54,12 @@ pub struct Cli {
 	#[structopt(raw = true)]
 	pub relaychain_args: Vec<String>,
 }
+
+#[derive(Debug, StructOpt, Clone)]
+pub struct PolkadotCli {
+	#[structopt(flatten)]
+	pub base: polkadot_cli::RunCmd,
+
+	#[structopt(skip)]
+	pub base_path: Option<PathBuf>,
+}

cumulus/test/parachain/src/command.rs

@@ -16,50 +16,123 @@
 
 use crate::chain_spec;
 use crate::cli::{Cli, PolkadotCli, Subcommand};
-
+use codec::Encode;
-use std::sync::Arc;
+use log::info;
-
 use parachain_runtime::Block;
-
+use sc_cli::{
+	CliConfiguration, Error, ImportParams, KeystoreParams, NetworkParams, Result, SharedParams,
+	SubstrateCli,
+};
 use sc_client::genesis;
-use sc_service::{Configuration, Role as ServiceRole, config::PrometheusConfig};
+use sc_network::config::TransportConfig;
+use sc_service::{
+	config::{NetworkConfiguration, NodeKeyConfig, PrometheusConfig},
+	Configuration, Role as ServiceRole,
+};
 use sp_core::hexdisplay::HexDisplay;
 use sp_runtime::{
 	traits::{Block as BlockT, Hash as HashT, Header as HeaderT},
 	BuildStorage,
 };
-use sc_network::config::TransportConfig;
+use std::net::SocketAddr;
+use std::path::PathBuf;
+use std::sync::Arc;
 
-use codec::Encode;
+impl SubstrateCli for Cli {
-use log::info;
+	fn impl_name() -> &'static str {
+		"Cumulus Test Parachain Collator"
+	}
 
-const DEFAULT_POLKADOT_RPC_HTTP: &'static str = "127.0.0.1:9934";
+	fn impl_version() -> &'static str {
-const DEFAULT_POLKADOT_RPC_WS: &'static str = "127.0.0.1:9945";
+		env!("SUBSTRATE_CLI_IMPL_VERSION")
-const DEFAULT_POLKADOT_PROMETHEUS_PORT: &'static str = "127.0.0.1:9616";
+	}
+
+	fn description() -> &'static str {
+		"Cumulus test parachain collator\n\nThe command-line arguments provided first will be \
+		passed to the parachain node, while the arguments provided after -- will be passed \
+		to the relaychain node.\n\n\
+		cumulus-test-parachain-collator [parachain-args] -- [relaychain-args]"
+	}
+
+	fn author() -> &'static str {
+		env!("CARGO_PKG_AUTHORS")
+	}
+
+	fn support_url() -> &'static str {
+		"https://github.com/paritytech/cumulus/issues/new"
+	}
+
+	fn copyright_start_year() -> i32 {
+		2017
+	}
+
+	fn executable_name() -> &'static str {
+		"cumulus-test-parachain-collator"
+	}
+
+	fn load_spec(&self, _id: &str) -> std::result::Result<Box<dyn sc_service::ChainSpec>, String> {
+		Ok(Box::new(chain_spec::get_chain_spec()))
+	}
+}
+
+impl SubstrateCli for PolkadotCli {
+	fn impl_name() -> &'static str {
+		"Cumulus Test Parachain Collator"
+	}
+
+	fn impl_version() -> &'static str {
+		env!("SUBSTRATE_CLI_IMPL_VERSION")
+	}
+
+	fn description() -> &'static str {
+		"Cumulus test parachain collator\n\nThe command-line arguments provided first will be \
+		passed to the parachain node, while the arguments provided after -- will be passed \
+		to the relaychain node.\n\n\
+		cumulus-test-parachain-collator [parachain-args] -- [relaychain-args]"
+	}
+
+	fn author() -> &'static str {
+		env!("CARGO_PKG_AUTHORS")
+	}
+
+	fn support_url() -> &'static str {
+		"https://github.com/paritytech/cumulus/issues/new"
+	}
+
+	fn copyright_start_year() -> i32 {
+		2017
+	}
+
+	fn executable_name() -> &'static str {
+		"cumulus-test-parachain-collator"
+	}
+
+	fn load_spec(&self, _id: &str) -> std::result::Result<Box<dyn sc_service::ChainSpec>, String> {
+		polkadot_service::PolkadotChainSpec::from_json_bytes(
+			&include_bytes!("../res/polkadot_chainspec.json")[..],
+		)
+		.map(|r| Box::new(r) as Box<_>)
+	}
+}
 
 /// Parse command line arguments into service configuration.
-pub fn run(version: sc_cli::VersionInfo) -> sc_cli::Result<()> {
+pub fn run() -> Result<()> {
-	let opt: Cli = sc_cli::from_args(&version);
+	let cli = Cli::from_args();
 
-	let mut config = sc_service::Configuration::from_version(&version);
+	match &cli.subcommand {
-	let mut polkadot_config = Configuration::from_version(&version);
-
-	match opt.subcommand {
 		Some(Subcommand::Base(subcommand)) => {
-			subcommand.init(&version)?;
+			let runner = cli.create_runner(subcommand)?;
-			subcommand.update_config(&mut config, load_spec, &version)?;
+
-			subcommand.run(
+			runner.run_subcommand(subcommand, |config| Ok(new_full_start!(config).0))
-				config,
+		}
-				|config: Configuration| Ok(new_full_start!(config).0),
-			)
-		},
 		Some(Subcommand::ExportGenesisState(params)) => {
 			sc_cli::init_logger("");
 
 			let storage = (&chain_spec::get_chain_spec()).build_storage()?;
 
 			let child_roots = storage.children.iter().map(|(sk, child_content)| {
-				let state_root = <<<Block as BlockT>::Header as HeaderT>::Hashing as HashT>::trie_root(
+				let state_root =
+					<<<Block as BlockT>::Header as HeaderT>::Hashing as HashT>::trie_root(
 						child_content.data.clone().into_iter().collect(),
 					);
 				(sk.clone(), state_root.encode())
@@ -71,74 +144,211 @@ pub fn run(version: sc_cli::VersionInfo) -> sc_cli::Result<()> {
 
 			let header_hex = format!("0x{:?}", HexDisplay::from(&block.header().encode()));
 
-			if let Some(output) = params.output {
+			if let Some(output) = &params.output {
 				std::fs::write(output, header_hex)?;
 			} else {
 				println!("{}", header_hex);
 			}
 
 			Ok(())
-		},
+		}
 		None => {
-			opt.run.init(&version)?;
+			let runner = cli.create_runner(&cli.run)?;
-			opt.run.update_config(&mut config, load_spec, &version)?;
-
-			info!("{}", version.name);
-			info!("  version {}", config.full_version());
-			info!("  by {}, 2019", version.author);
-			info!("Chain specification: {}", config.expect_chain_spec().name());
-			info!("Node name: {}", config.name);
-			info!("Roles: {:?}", config.role);
-			info!("Parachain id: {:?}", crate::PARA_ID);
 
 			// TODO
 			let key = Arc::new(sp_core::Pair::from_seed(&[10; 32]));
+			let key2 = Arc::new(sp_core::Pair::from_seed(&[10; 32]));
 
-			polkadot_config.config_dir = config.in_chain_config_dir("polkadot");
+			let mut polkadot_cli = PolkadotCli::from_iter(
-
+				[PolkadotCli::executable_name().to_string()]
-			let polkadot_opt: PolkadotCli = sc_cli::from_iter(
+					.iter()
-				[version.executable_name.to_string()].iter().chain(opt.relaychain_args.iter()),
+					.chain(cli.relaychain_args.iter()),
-				&version,
 			);
-			let allow_private_ipv4 = !polkadot_opt.run.base.network_config.no_private_ipv4;
 
-			polkadot_config.rpc_http = Some(DEFAULT_POLKADOT_RPC_HTTP.parse().unwrap());
+			polkadot_cli.base_path = cli.run.base_path()?.map(|x| x.join("polkadot"));
-			polkadot_config.rpc_ws = Some(DEFAULT_POLKADOT_RPC_WS.parse().unwrap());
+
-			polkadot_config.prometheus_config = Some(
+			runner.run_node(
-				PrometheusConfig::new_with_default_registry(
+				|config| {
-					DEFAULT_POLKADOT_PROMETHEUS_PORT.parse().unwrap(),
+					let task_executor = config.task_executor.clone();
+					let polkadot_config = SubstrateCli::create_configuration(
+						&polkadot_cli,
+						&polkadot_cli,
+						task_executor,
 					)
-			);
+					.unwrap();
 
-			polkadot_opt.run.base.update_config(
+					info!("Parachain id: {:?}", crate::PARA_ID);
-				&mut polkadot_config,
+
-				load_spec_polkadot,
+					crate::service::run_collator(config, key2, polkadot_config)
-				&version,
+				},
-			)?;
+				|config| {
+					let task_executor = config.task_executor.clone();
+					let polkadot_config = SubstrateCli::create_configuration(
+						&polkadot_cli,
+						&polkadot_cli,
+						task_executor,
+					)
+					.unwrap();
+
+					info!("Parachain id: {:?}", crate::PARA_ID);
+
+					crate::service::run_collator(config, key, polkadot_config)
+				},
+			)
+		}
+	}
+}
+
+impl CliConfiguration for PolkadotCli {
+	fn shared_params(&self) -> &SharedParams {
+		self.base.base.shared_params()
+	}
+
+	fn import_params(&self) -> Option<&ImportParams> {
+		self.base.base.import_params()
+	}
+
+	fn network_params(&self) -> Option<&NetworkParams> {
+		self.base.base.network_params()
+	}
+
+	fn keystore_params(&self) -> Option<&KeystoreParams> {
+		self.base.base.keystore_params()
+	}
+
+	fn base_path(&self) -> Result<Option<PathBuf>> {
+		Ok(self.shared_params().base_path().or(self.base_path.clone()))
+	}
+
+	fn rpc_http(&self) -> Result<Option<SocketAddr>> {
+		let rpc_external = self.base.base.rpc_external;
+		let unsafe_rpc_external = self.base.base.unsafe_rpc_external;
+		let validator = self.base.base.validator;
+		let rpc_port = self.base.base.rpc_port;
+		// copied directly from substrate
+		let rpc_interface: &str = interface_str(rpc_external, unsafe_rpc_external, validator)?;
+
+		Ok(Some(parse_address(
+			&format!("{}:{}", rpc_interface, 9934),
+			rpc_port,
+		)?))
+	}
+
+	fn rpc_ws(&self) -> Result<Option<SocketAddr>> {
+		let ws_external = self.base.base.ws_external;
+		let unsafe_ws_external = self.base.base.unsafe_ws_external;
+		let validator = self.base.base.validator;
+		let ws_port = self.base.base.ws_port;
+		// copied directly from substrate
+		let ws_interface: &str = interface_str(ws_external, unsafe_ws_external, validator)?;
+
+		Ok(Some(parse_address(
+			&format!("{}:{}", ws_interface, 9945),
+			ws_port,
+		)?))
+	}
+
+	fn prometheus_config(&self) -> Result<Option<PrometheusConfig>> {
+		let no_prometheus = self.base.base.no_prometheus;
+		let prometheus_external = self.base.base.prometheus_external;
+		let prometheus_port = self.base.base.prometheus_port;
+
+		if no_prometheus {
+			Ok(None)
+		} else {
+			let prometheus_interface: &str = if prometheus_external {
+				"0.0.0.0"
+			} else {
+				"127.0.0.1"
+			};
+
+			Ok(Some(PrometheusConfig::new_with_default_registry(
+				parse_address(
+					&format!("{}:{}", prometheus_interface, 9616),
+					prometheus_port,
+				)?,
+			)))
+		}
+	}
 
 	// TODO: we disable mdns for the polkadot node because it prevents the process to exit
 	//       properly. See https://github.com/paritytech/cumulus/issues/57
-			polkadot_config.network.transport = TransportConfig::Normal {
+	fn network_config(
+		&self,
+		chain_spec: &Box<dyn sc_service::ChainSpec>,
+		is_dev: bool,
+		net_config_dir: &PathBuf,
+		client_id: &str,
+		node_name: &str,
+		node_key: NodeKeyConfig,
+	) -> Result<NetworkConfiguration> {
+		let (mut network, allow_private_ipv4) = self
+			.network_params()
+			.map(|x| {
+				(
+					x.network_config(
+						chain_spec,
+						is_dev,
+						net_config_dir,
+						client_id,
+						node_name,
+						node_key,
+					),
+					!x.no_private_ipv4,
+				)
+			})
+			.expect("NetworkParams is always available on RunCmd; qed");
+
+		network.transport = TransportConfig::Normal {
 			enable_mdns: false,
 			allow_private_ipv4,
 			wasm_external_transport: None,
 			use_yamux_flow_control: false,
 		};
 
-			match config.role {
+		Ok(network)
-				ServiceRole::Light => unimplemented!("Light client not supported!"),
-				_ => crate::service::run_collator(config, key, polkadot_config),
 	}
-		},
+
+	fn init<C: SubstrateCli>(&self) -> Result<()> {
+		unreachable!("PolkadotCli is never initialized; qed");
 	}
 }
 
-fn load_spec(_: &str) -> Result<Box<dyn sc_service::ChainSpec>, String> {
+// copied directly from substrate
-	Ok(Box::new(chain_spec::get_chain_spec()))
+fn parse_address(address: &str, port: Option<u16>) -> std::result::Result<SocketAddr, String> {
+	let mut address: SocketAddr = address
+		.parse()
+		.map_err(|_| format!("Invalid address: {}", address))?;
+	if let Some(port) = port {
+		address.set_port(port);
+	}
+
+	Ok(address)
 }
 
-fn load_spec_polkadot(_: &str) -> Result<Box<dyn sc_service::ChainSpec>, String> {
+// copied directly from substrate
-	polkadot_service::PolkadotChainSpec::from_json_bytes(
+fn interface_str(
-		&include_bytes!("../res/polkadot_chainspec.json")[..],
+	is_external: bool,
-	).map(|r| Box::new(r) as Box<_>)
+	is_unsafe_external: bool,
+	is_validator: bool,
+) -> Result<&'static str> {
+	if is_external && is_validator {
+		return Err(Error::Input(
+			"--rpc-external and --ws-external options shouldn't be \
+		used if the node is running as a validator. Use `--unsafe-rpc-external` if you understand \
+		the risks. See the options description for more information."
+				.to_owned(),
+		));
+	}
+
+	if is_external || is_unsafe_external {
+		log::warn!(
+			"It isn't safe to expose RPC publicly without a proxy server that filters \
+		available set of RPC methods."
+		);
+
+		Ok("0.0.0.0")
+	} else {
+		Ok("127.0.0.1")
+	}
 }

cumulus/test/parachain/src/main.rs

@@ -29,24 +29,7 @@ mod command;
 
 /// The parachain id of this parachain.
 pub const PARA_ID: ParaId = ParaId::new(100);
-const EXECUTABLE_NAME: &'static str = "cumulus-test-parachain-collator";
-const DESCRIPTION: &'static str =
-	"Cumulus test parachain collator\n\nThe command-line arguments provided first will be \
-	passed to the parachain node, while the arguments provided after -- will be passed \
-	to the relaychain node.\n\n\
-	cumulus-test-parachain-collator [parachain-args] -- [relaychain-args]";
 
 fn main() -> sc_cli::Result<()> {
-	let version = sc_cli::VersionInfo {
+	command::run()
-		name: "Cumulus Test Parachain Collator",
-		commit: env!("VERGEN_SHA_SHORT"),
-		version: env!("CARGO_PKG_VERSION"),
-		author: "Parity Technologies <admin@parity.io>",
-		description: DESCRIPTION,
-		executable_name: EXECUTABLE_NAME,
-		support_url: "https://github.com/paritytech/cumulus/issues/new",
-		copyright_start_year: 2017,
-	};
-
-	command::run(version)
 }

cumulus/test/parachain/src/service.rs

@@ -74,11 +74,8 @@ macro_rules! new_full_start {
 pub fn run_collator(
 	parachain_config: Configuration,
 	key: Arc<CollatorPair>,
-	mut polkadot_config: polkadot_collator::Configuration,
+	polkadot_config: polkadot_collator::Configuration,
-) -> sc_cli::Result<()> {
+) -> sc_service::error::Result<impl AbstractService> {
-	sc_cli::run_service_until_exit(parachain_config, move |parachain_config| {
-		polkadot_config.task_executor = parachain_config.task_executor.clone();
-
 	let (builder, inherent_data_providers) = new_full_start!(parachain_config);
 	inherent_data_providers
 		.register_provider(sp_timestamp::InherentDataProvider)
@@ -116,5 +113,4 @@ pub fn run_collator(
 	service.spawn_essential_task("polkadot", polkadot_future);
 
 	Ok(service)
-	})
 }

cumulus/test/parachain/tests/polkadot_argument_parsing.rs

@@ -32,9 +32,11 @@ fn polkadot_argument_parsing() {
 		let _ = fs::remove_dir_all("polkadot_argument_parsing");
 		let mut cmd = Command::new(cargo_bin("cumulus-test-parachain-collator"))
 			.args(&[
+				"--dev",
 				"-d",
 				"polkadot_argument_parsing",
 				"--",
+				"--dev",
 				"--bootnodes",
 				"/ip4/127.0.0.1/tcp/30333/p2p/Qmbx43psh7LVkrYTRXisUpzCubbgYojkejzAgj5mteDnxy",
 				"--bootnodes",