Add a build-sync-spec subcommand and remove the CHT roots from the light sync state. (#6999)

* Move subcommands from sc-cli to nodes

* Add --build-sync-spec subcommand

* Remove CHTs from snapshots

* Keep ProvideChtRoots

substrate/bin/node/cli/src/chain_spec.rs

@@ -380,7 +380,7 @@ pub fn local_testnet_config() -> ChainSpec {
 #[cfg(test)]
 pub(crate) mod tests {
 	use super::*;
-	use crate::service::{new_full_base, new_light_base};
+	use crate::service::{new_full_base, new_light_base, NewFullBase};
 	use sc_service_test;
 	use sp_runtime::BuildStorage;
 
@@ -431,8 +431,9 @@ pub(crate) mod tests {
 		sc_service_test::connectivity(
 			integration_test_config_with_two_authorities(),
 			|config| {
-				let (keep_alive, _, client, network, transaction_pool) = new_full_base(config,|_, _| ())?;
-				Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool))
+				let NewFullBase { task_manager, client, network, transaction_pool, .. }
+					= new_full_base(config,|_, _| ())?;
+				Ok(sc_service_test::TestNetComponents::new(task_manager, client, network, transaction_pool))
 			},
 			|config| {
 				let (keep_alive, _, client, network, transaction_pool) = new_light_base(config)?;

substrate/bin/node/cli/src/cli.rs

@@ -59,6 +59,9 @@ pub enum Subcommand {
 	/// Build a chain specification.
 	BuildSpec(sc_cli::BuildSpecCmd),
 
+	/// Build a chain specification with a light client sync state.
+	BuildSyncSpec(sc_cli::BuildSyncSpecCmd),
+
 	/// Validate blocks.
 	CheckBlock(sc_cli::CheckBlockCmd),
 

substrate/bin/node/cli/src/command.rs

@@ -21,7 +21,7 @@ use node_executor::Executor;
 use node_runtime::{Block, RuntimeApi};
 use sc_cli::{Result, SubstrateCli, RuntimeVersion, Role, ChainSpec};
 use sc_service::PartialComponents;
-use crate::service::new_partial;
+use crate::service::{new_partial, new_full_base, NewFullBase};
 
 impl SubstrateCli for Cli {
 	fn impl_name() -> String {
@@ -101,6 +101,17 @@ pub fn run() -> Result<()> {
 			let runner = cli.create_runner(cmd)?;
 			runner.sync_run(|config| cmd.run(config.chain_spec, config.network))
 		},
+		Some(Subcommand::BuildSyncSpec(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let chain_spec = config.chain_spec.cloned_box();
+				let network_config = config.network.clone();
+				let NewFullBase { task_manager, client, network_status_sinks, .. }
+					= new_full_base(config, |_, _| ())?;
+
+				Ok((cmd.run(chain_spec, network_config, client, network_status_sinks), task_manager))
+			})
+		},
 		Some(Subcommand::CheckBlock(cmd)) => {
 			let runner = cli.create_runner(cmd)?;
 			runner.async_run(|config| {

substrate/bin/node/cli/src/service.rs

@@ -151,6 +151,15 @@ pub fn new_partial(config: &Configuration) -> Result<sc_service::PartialComponen
 	})
 }
 
+pub struct NewFullBase {
+	pub task_manager: TaskManager,
+	pub inherent_data_providers: InherentDataProviders,
+	pub client: Arc<FullClient>,
+	pub network: Arc<NetworkService<Block, <Block as BlockT>::Hash>>,
+	pub network_status_sinks: sc_service::NetworkStatusSinks<Block>,
+	pub transaction_pool: Arc<sc_transaction_pool::FullPool<Block, FullClient>>,
+}
+
 /// Creates a full service from the configuration.
 pub fn new_full_base(
 	config: Configuration,
@@ -158,11 +167,7 @@ pub fn new_full_base(
 		&sc_consensus_babe::BabeBlockImport<Block, FullClient, FullGrandpaBlockImport>,
 		&sc_consensus_babe::BabeLink<Block>,
 	)
-) -> Result<(
-	TaskManager, InherentDataProviders, Arc<FullClient>,
-	Arc<NetworkService<Block, <Block as BlockT>::Hash>>,
-	Arc<sc_transaction_pool::FullPool<Block, FullClient>>,
-), ServiceError> {
+) -> Result<NewFullBase, ServiceError> {
 	let sc_service::PartialComponents {
 		client, backend, mut task_manager, import_queue, keystore, select_chain, transaction_pool,
 		inherent_data_providers,
@@ -210,7 +215,7 @@ pub fn new_full_base(
 		on_demand: None,
 		remote_blockchain: None,
 		telemetry_connection_sinks: telemetry_connection_sinks.clone(),
-		network_status_sinks,
+		network_status_sinks: network_status_sinks.clone(),
 		system_rpc_tx,
 	})?;
 
@@ -330,13 +335,16 @@ pub fn new_full_base(
 	}
 
 	network_starter.start_network();
-	Ok((task_manager, inherent_data_providers, client, network, transaction_pool))
+	Ok(NewFullBase {
+		task_manager, inherent_data_providers, client, network, network_status_sinks,
+		transaction_pool,
+	})
 }
 
 /// Builds a new service for a full client.
 pub fn new_full(config: Configuration)
 -> Result<TaskManager, ServiceError> {
-	new_full_base(config, |_, _| ()).map(|(task_manager, _, _, _, _)| {
+	new_full_base(config, |_, _| ()).map(|NewFullBase { task_manager, .. }| {
 		task_manager
 	})
 }
@@ -467,7 +475,7 @@ mod tests {
 	use sp_finality_tracker;
 	use sp_keyring::AccountKeyring;
 	use sc_service_test::TestNetNode;
-	use crate::service::{new_full_base, new_light_base};
+	use crate::service::{new_full_base, new_light_base, NewFullBase};
 	use sp_runtime::traits::IdentifyAccount;
 	use sp_transaction_pool::{MaintainedTransactionPool, ChainEvent};
 	use sc_client_api::BlockBackend;
@@ -499,18 +507,19 @@ mod tests {
 			chain_spec,
 			|config| {
 				let mut setup_handles = None;
-				let (keep_alive, inherent_data_providers, client, network, transaction_pool) =
-					new_full_base(config,
-						|
-							block_import: &sc_consensus_babe::BabeBlockImport<Block, _, _>,
-							babe_link: &sc_consensus_babe::BabeLink<Block>,
-						| {
-							setup_handles = Some((block_import.clone(), babe_link.clone()));
-						}
-					)?;
+				let NewFullBase {
+					task_manager, inherent_data_providers, client, network, transaction_pool, ..
+				} = new_full_base(config,
+					|
+						block_import: &sc_consensus_babe::BabeBlockImport<Block, _, _>,
+						babe_link: &sc_consensus_babe::BabeLink<Block>,
+					| {
+						setup_handles = Some((block_import.clone(), babe_link.clone()));
+					}
+				)?;
 
 				let node = sc_service_test::TestNetComponents::new(
-					keep_alive, client, network, transaction_pool
+					task_manager, client, network, transaction_pool
 				);
 				Ok((node, (inherent_data_providers, setup_handles.unwrap())))
 			},
@@ -661,8 +670,9 @@ mod tests {
 		sc_service_test::consensus(
 			crate::chain_spec::tests::integration_test_config_with_two_authorities(),
 			|config| {
-				let (keep_alive, _, client, network, transaction_pool) = new_full_base(config, |_, _| ())?;
-				Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool))
+				let NewFullBase { task_manager, client, network, transaction_pool, .. }
+					= new_full_base(config,|_, _| ())?;
+				Ok(sc_service_test::TestNetComponents::new(task_manager, client, network, transaction_pool))
 			},
 			|config| {
 				let (keep_alive, _, client, network, transaction_pool) = new_light_base(config)?;

substrate/client/chain-spec/src/chain_spec.rs

@@ -401,8 +401,6 @@ where
 pub struct LightSyncState<Block: BlockT> {
 	/// The header of the best finalized block.
 	pub header: <Block as BlockT>::Header,
-	/// A list of all CHTs in the chain.
-	pub chts: Vec<<Block as BlockT>::Hash>,
 }
 
 impl<Block: BlockT> LightSyncState<Block> {
@@ -412,7 +410,6 @@ impl<Block: BlockT> LightSyncState<Block> {
 
 		SerializableLightSyncState {
 			header: StorageData(self.header.encode()),
-			chts: self.chts.iter().map(|hash| StorageData(hash.encode())).collect(),
 		}
 	}
 
@@ -420,9 +417,6 @@ impl<Block: BlockT> LightSyncState<Block> {
 	pub fn from_serializable(serialized: &SerializableLightSyncState) -> Result<Self, codec::Error> {
 		Ok(Self {
 			header: codec::Decode::decode(&mut &serialized.header.0[..])?,
-			chts: serialized.chts.iter()
-				.map(|cht| codec::Decode::decode(&mut &cht.0[..]))
-				.collect::<Result<_, _>>()?,
 		})
 	}
 }
@@ -433,7 +427,6 @@ impl<Block: BlockT> LightSyncState<Block> {
 #[serde(deny_unknown_fields)]
 pub struct SerializableLightSyncState {
 	header: StorageData,
-	chts: Vec<StorageData>,
 }
 
 #[cfg(test)]

substrate/client/cli/src/commands/build_sync_spec_cmd.rs

@@ -0,0 +1,113 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2018-2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::error;
+use crate::params::{SharedParams, NetworkParams};
+use crate::CliConfiguration;
+use log::info;
+use sc_network::config::build_multiaddr;
+use sc_service::{config::{MultiaddrWithPeerId, NetworkConfiguration}, ChainSpec};
+use structopt::StructOpt;
+use std::io::Write;
+use std::sync::Arc;
+use sp_runtime::traits::Block as BlockT;
+use sc_service::chain_ops::build_light_sync_state;
+use sc_service::NetworkStatusSinks;
+use futures::{FutureExt, StreamExt};
+use futures::future::ready;
+
+/// The `build-sync-spec` command used to build a chain spec that contains a light client state
+/// so that light clients can sync faster.
+#[derive(Debug, StructOpt)]
+pub struct BuildSyncSpecCmd {
+	/// Force raw genesis storage output.
+	#[structopt(long = "raw")]
+	pub raw: bool,
+
+	/// Sync the chain using a full client first.
+	#[structopt(long)]
+	pub sync_first: bool,
+
+	/// Disable adding the default bootnode to the specification.
+	///
+	/// By default the `/ip4/127.0.0.1/tcp/30333/p2p/NODE_PEER_ID` bootnode is added to the
+	/// specification when no bootnode exists.
+	#[structopt(long = "disable-default-bootnode")]
+	pub disable_default_bootnode: bool,
+
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub shared_params: SharedParams,
+
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub network_params: NetworkParams,
+}
+
+impl BuildSyncSpecCmd {
+	/// Run the build-sync-spec command
+	pub async fn run<B, CL>(
+		&self,
+		mut spec: Box<dyn ChainSpec>,
+		network_config: NetworkConfiguration,
+		client: Arc<CL>,
+		network_status_sinks: NetworkStatusSinks<B>,
+	) -> error::Result<()>
+		where
+			B: BlockT,
+			CL: sp_blockchain::HeaderBackend<B>,
+	{
+        if self.sync_first {
+            network_status_sinks.network_status(std::time::Duration::from_secs(1)).filter(|(status, _)| {
+                ready(status.sync_state == sc_network::SyncState::Idle && status.num_sync_peers > 0)
+            }).into_future().map(drop).await;
+        }
+
+        let light_sync_state = build_light_sync_state(client)?;
+        spec.set_light_sync_state(light_sync_state.to_serializable());
+
+		info!("Building chain spec");
+		let raw_output = self.raw;
+
+		if spec.boot_nodes().is_empty() && !self.disable_default_bootnode {
+			let keys = network_config.node_key.into_keypair()?;
+			let peer_id = keys.public().into_peer_id();
+			let addr = MultiaddrWithPeerId {
+				multiaddr: build_multiaddr![Ip4([127, 0, 0, 1]), Tcp(30333u16)],
+				peer_id,
+			};
+			spec.add_boot_node(addr)
+		}
+
+		let json = sc_service::chain_ops::build_spec(&*spec, raw_output)?;
+		if std::io::stdout().write_all(json.as_bytes()).is_err() {
+			let _ = std::io::stderr().write_all(b"Error writing to stdout\n");
+		}
+		Ok(())
+	}
+}
+
+impl CliConfiguration for BuildSyncSpecCmd {
+	fn shared_params(&self) -> &SharedParams {
+		&self.shared_params
+	}
+
+	fn network_params(&self) -> Option<&NetworkParams> {
+		Some(&self.network_params)
+	}
+}

substrate/client/cli/src/commands/mod.rs

@@ -16,6 +16,7 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 mod build_spec_cmd;
+mod build_sync_spec_cmd;
 mod check_block_cmd;
 mod export_blocks_cmd;
 mod export_state_cmd;
@@ -36,6 +37,7 @@ pub mod utils;
 
 pub use self::{
 	build_spec_cmd::BuildSpecCmd,
+	build_sync_spec_cmd::BuildSyncSpecCmd,
 	check_block_cmd::CheckBlockCmd,
 	export_blocks_cmd::ExportBlocksCmd,
 	export_state_cmd::ExportStateCmd,

substrate/client/service/src/chain_ops/build_sync_spec.rs

@@ -14,48 +14,23 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
-use sp_runtime::traits::{Block as BlockT, NumberFor, Saturating, One};
+use sp_runtime::traits::Block as BlockT;
 use sp_blockchain::HeaderBackend;
 use std::sync::Arc;
 use sp_runtime::generic::BlockId;
-use sc_client_api::ProvideChtRoots;
 
 /// Build a `LightSyncState` from the CHT roots stored in a backend.
-pub fn build_light_sync_state<TBl, TCl, TBackend>(
+pub fn build_light_sync_state<TBl, TCl>(
 	client: Arc<TCl>,
-	backend: Arc<TBackend>,
 ) -> Result<sc_chain_spec::LightSyncState<TBl>, sp_blockchain::Error>
 	where
 		TBl: BlockT,
 		TCl: HeaderBackend<TBl>,
-		TBackend: sc_client_api::Backend<TBl>,
-		<TBackend as sc_client_api::Backend<TBl>>::Blockchain: ProvideChtRoots<TBl>,
 {
-	let cht_root_provider = backend.blockchain();
-
 	let finalized_hash = client.info().finalized_hash;
-	let finalized_number = client.info().finalized_number;
-
-	use sc_client_api::cht;
-
-	let mut chts = Vec::new();
-
-	// We can't fetch a CHT root later than `finalized_number - 2 * cht_size`.
-	let cht_size_x_2 = cht::size::<NumberFor::<TBl>>() * NumberFor::<TBl>::from(2);
-
-	let mut number = NumberFor::<TBl>::one();
-
-	while number <= finalized_number.saturating_sub(cht_size_x_2) {
-		match cht_root_provider.header_cht_root(cht::size(), number)? {
-			Some(cht_root) => chts.push(cht_root),
-			None => log::error!("No CHT found for block {}", number),
-		}
-
-		number += cht::size();
-	}
+	let header = client.header(BlockId::Hash(finalized_hash))?.unwrap();
 
 	Ok(sc_chain_spec::LightSyncState {
-		header: client.header(BlockId::Hash(finalized_hash))?.unwrap(),
-		chts,
+		header
 	})
 }

substrate/client/service/src/chain_ops/mod.rs

@@ -21,11 +21,11 @@ mod export_blocks;
 mod export_raw_state;
 mod import_blocks;
 mod revert_chain;
-mod build_spec;
+mod build_sync_spec;
 
 pub use check_block::*;
 pub use export_blocks::*;
 pub use export_raw_state::*;
 pub use import_blocks::*;
 pub use revert_chain::*;
-pub use build_spec::*;
+pub use build_sync_spec::*;