Add a LightSyncState field to the chain spec (#6894)

* Reset code, almost ready for PR

* Improved build_hardcoded_spec

* Fix line widths

* Fix tests

* Fix sc-service-test

* Suggestions from code review

* Rename to LightSyncState

* It's not syncing :^(

* It syncs!

* Remove rpc call

* Convert spaces to tabs

* Moved sc-service things to export_sync_state.rs

* Fix tests

* Wait for syncing with network_status_sinks

* Remove sc-network from node-template

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Various changes, split the flag up into 2 pieces to make testing easier.

* Update client/cli/src/commands/build_spec_cmd.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Revert a lot of changes

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

substrate/Cargo.lock

@@ -6324,6 +6324,7 @@ name = "sc-chain-spec"
 version = "2.0.0-rc6"
 dependencies = [
  "impl-trait-for-tuples",
+ "parity-scale-codec",
  "sc-chain-spec-derive",
  "sc-network",
  "sc-telemetry",

substrate/client/api/src/in_mem.rs

@@ -447,6 +447,16 @@ impl<Block: BlockT> light::Storage<Block> for Blockchain<Block>
 		Blockchain::finalize_header(self, id, None)
 	}
 
+	fn cache(&self) -> Option<Arc<dyn blockchain::Cache<Block>>> {
+		None
+	}
+
+	fn usage_info(&self) -> Option<UsageInfo> {
+		None
+	}
+}
+
+impl<Block: BlockT> light::ChtRootStorage<Block> for Blockchain<Block> {
 	fn header_cht_root(
 		&self,
 		_cht_size: NumberFor<Block>,
@@ -466,14 +476,6 @@ impl<Block: BlockT> light::Storage<Block> for Blockchain<Block>
 			.ok_or_else(|| sp_blockchain::Error::Backend(format!("Changes trie CHT for block {} not exists", block)))
 			.map(Some)
 	}
-
-	fn cache(&self) -> Option<Arc<dyn blockchain::Cache<Block>>> {
-		None
-	}
-
-	fn usage_info(&self) -> Option<UsageInfo> {
-		None
-	}
 }
 
 /// In-memory operation.

substrate/client/api/src/light.rs

@@ -232,7 +232,9 @@ pub trait FetchChecker<Block: BlockT>: Send + Sync {
 
 
 /// Light client blockchain storage.
-pub trait Storage<Block: BlockT>: AuxStore + HeaderBackend<Block> + HeaderMetadata<Block, Error=ClientError> {
+pub trait Storage<Block: BlockT>: AuxStore + HeaderBackend<Block>
+	+ HeaderMetadata<Block, Error=ClientError> + ChtRootStorage<Block>
+{
 	/// Store new header. Should refuse to revert any finalized blocks.
 	///
 	/// Takes new authorities, the leaf state of the new block, and
@@ -254,6 +256,15 @@ pub trait Storage<Block: BlockT>: AuxStore + HeaderBackend<Block> + HeaderMetada
 	/// Get last finalized header.
 	fn last_finalized(&self) -> ClientResult<Block::Hash>;
 
+	/// Get storage cache.
+	fn cache(&self) -> Option<Arc<dyn BlockchainCache<Block>>>;
+
+	/// Get storage usage statistics.
+	fn usage_info(&self) -> Option<UsageInfo>;
+}
+
+/// Light client CHT root storage.
+pub trait ChtRootStorage<Block: BlockT> {
 	/// Get headers CHT root for given block. Returns None if the block is not pruned (not a part of any CHT).
 	fn header_cht_root(
 		&self,
@@ -267,12 +278,6 @@ pub trait Storage<Block: BlockT>: AuxStore + HeaderBackend<Block> + HeaderMetada
 		cht_size: NumberFor<Block>,
 		block: NumberFor<Block>,
 	) -> ClientResult<Option<Block::Hash>>;
-
-	/// Get storage cache.
-	fn cache(&self) -> Option<Arc<dyn BlockchainCache<Block>>>;
-
-	/// Get storage usage statistics.
-	fn usage_info(&self) -> Option<UsageInfo>;
 }
 
 /// Remote header.

substrate/client/chain-spec/Cargo.toml

@@ -21,3 +21,4 @@ serde_json = "1.0.41"
 sp-runtime = { version = "2.0.0-rc6", path = "../../primitives/runtime" }
 sp-chain-spec = { version = "2.0.0-rc6", path = "../../primitives/chain-spec" }
 sc-telemetry = { version = "2.0.0-rc6", path = "../telemetry" }
+codec = { package = "parity-scale-codec", version = "1.3.4" }

substrate/client/chain-spec/src/chain_spec.rs

@@ -17,6 +17,7 @@
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 //! Substrate chain configurations.
+#![warn(missing_docs)]
 
 use std::{borrow::Cow, fs::File, path::PathBuf, sync::Arc, collections::HashMap};
 use serde::{Serialize, Deserialize};
@@ -26,6 +27,7 @@ use serde_json as json;
 use crate::{RuntimeGenesis, ChainType, extension::GetExtension, Properties};
 use sc_network::config::MultiaddrWithPeerId;
 use sc_telemetry::TelemetryEndpoints;
+use sp_runtime::traits::Block as BlockT;
 
 enum GenesisSource<G> {
 	File(PathBuf),
@@ -157,6 +159,7 @@ struct ClientSpec<E> {
 	consensus_engine: (),
 	#[serde(skip_serializing)]
 	genesis: serde::de::IgnoredAny,
+	light_sync_state: Option<SerializableLightSyncState>,
 }
 
 /// A type denoting empty extensions.
@@ -245,6 +248,7 @@ impl<G, E> ChainSpec<G, E> {
 			extensions,
 			consensus_engine: (),
 			genesis: Default::default(),
+			light_sync_state: None,
 		};
 
 		ChainSpec {
@@ -257,6 +261,11 @@ impl<G, E> ChainSpec<G, E> {
 	fn chain_type(&self) -> ChainType {
 		self.client_spec.chain_type.clone()
 	}
+
+	/// Hardcode infomation to allow light clients to sync quickly into the chain spec.
+	fn set_light_sync_state(&mut self, light_sync_state: SerializableLightSyncState) {
+		self.client_spec.light_sync_state = Some(light_sync_state);	
+	}
 }
 
 impl<G, E: serde::de::DeserializeOwned> ChainSpec<G, E> {
@@ -284,16 +293,15 @@ impl<G, E: serde::de::DeserializeOwned> ChainSpec<G, E> {
 	}
 }
 
-impl<G: RuntimeGenesis, E: serde::Serialize + Clone + 'static> ChainSpec<G, E> {
-	/// Dump to json string.
-	pub fn as_json(&self, raw: bool) -> Result<String, String> {
-		#[derive(Serialize, Deserialize)]
-		struct Container<G, E> {
-			#[serde(flatten)]
-			client_spec: ClientSpec<E>,
-			genesis: Genesis<G>,
+#[derive(Serialize, Deserialize)]
+struct JsonContainer<G, E> {
+	#[serde(flatten)]
+	client_spec: ClientSpec<E>,
+	genesis: Genesis<G>,
+}
 
-		};
+impl<G: RuntimeGenesis, E: serde::Serialize + Clone + 'static> ChainSpec<G, E> {
+	fn json_container(&self, raw: bool) -> Result<JsonContainer<G, E>, String> {
 		let genesis = match (raw, self.genesis.resolve()?) {
 			(true, Genesis::Runtime(g)) => {
 				let storage = g.build_storage()?;
@@ -313,10 +321,15 @@ impl<G: RuntimeGenesis, E: serde::Serialize + Clone + 'static> ChainSpec<G, E> {
 			},
 			(_, genesis) => genesis,
 		};
-		let container = Container {
+		Ok(JsonContainer {
 			client_spec: self.client_spec.clone(),
 			genesis,
-		};
+		})
+	}
+
+	/// Dump to json string.
+	pub fn as_json(&self, raw: bool) -> Result<String, String> {
+		let container = self.json_container(raw)?;
 		json::to_string_pretty(&container)
 			.map_err(|e| format!("Error generating spec json: {}", e))
 	}
@@ -378,6 +391,49 @@ where
 	fn set_storage(&mut self, storage: Storage) {
 		self.genesis = GenesisSource::Storage(storage);
 	}
+
+	fn set_light_sync_state(&mut self, light_sync_state: SerializableLightSyncState) {
+		ChainSpec::set_light_sync_state(self, light_sync_state)
+	}
+}
+
+/// Hardcoded infomation that allows light clients to sync quickly.
+pub struct LightSyncState<Block: BlockT> {
+	/// The header of the best finalized block.
+	pub header: <Block as BlockT>::Header,
+	/// A list of all CHTs in the chain.
+	pub chts: Vec<<Block as BlockT>::Hash>,
+}
+
+impl<Block: BlockT> LightSyncState<Block> {
+	/// Convert into a `SerializableLightSyncState`.
+	pub fn to_serializable(&self) -> SerializableLightSyncState {
+		use codec::Encode;
+
+		SerializableLightSyncState {
+			header: StorageData(self.header.encode()),
+			chts: self.chts.iter().map(|hash| StorageData(hash.encode())).collect(),
+		}
+	}
+
+	/// Convert from a `SerializableLightSyncState`.
+	pub fn from_serializable(serialized: &SerializableLightSyncState) -> Result<Self, codec::Error> {
+		Ok(Self {
+			header: codec::Decode::decode(&mut &serialized.header.0[..])?,
+			chts: serialized.chts.iter()
+				.map(|cht| codec::Decode::decode(&mut &cht.0[..]))
+				.collect::<Result<_, _>>()?,
+		})
+	}
+}
+
+/// The serializable form of `LightSyncState`. Created using `LightSyncState::serialize`.
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+#[serde(deny_unknown_fields)]
+pub struct SerializableLightSyncState {
+	header: StorageData,
+	chts: Vec<StorageData>,
 }
 
 #[cfg(test)]

substrate/client/chain-spec/src/lib.rs

@@ -108,7 +108,9 @@
 mod chain_spec;
 mod extension;
 
-pub use chain_spec::{ChainSpec as GenericChainSpec, NoExtension};
+pub use chain_spec::{
+	ChainSpec as GenericChainSpec, NoExtension, LightSyncState, SerializableLightSyncState,
+};
 pub use extension::{Group, Fork, Forks, Extension, GetExtension, get_extension};
 pub use sc_chain_spec_derive::{ChainSpecExtension, ChainSpecGroup};
 pub use sp_chain_spec::{Properties, ChainType};
@@ -155,6 +157,8 @@ pub trait ChainSpec: BuildStorage + Send {
 	///
 	/// This will be used as storage at genesis.
 	fn set_storage(&mut self, storage: Storage);
+	/// Hardcode infomation to allow light clients to sync quickly into the chain spec.
+	fn set_light_sync_state(&mut self, light_sync_state: SerializableLightSyncState);
 }
 
 impl std::fmt::Debug for dyn ChainSpec {

substrate/client/db/src/light.rs

@@ -27,7 +27,7 @@ use sc_client_api::{
 	blockchain::{
 		BlockStatus, Cache as BlockchainCache, Info as BlockchainInfo,
 	},
-	Storage
+	Storage, ChtRootStorage,
 };
 use sp_blockchain::{
 	CachedHeaderMetadata, HeaderMetadata, HeaderMetadataCache,
@@ -523,22 +523,6 @@ impl<Block> Storage<Block> for LightStorage<Block>
 		}
 	}
 
-	fn header_cht_root(
-		&self,
-		cht_size: NumberFor<Block>,
-		block: NumberFor<Block>,
-	) -> ClientResult<Option<Block::Hash>> {
-		self.read_cht_root(HEADER_CHT_PREFIX, cht_size, block)
-	}
-
-	fn changes_trie_cht_root(
-		&self,
-		cht_size: NumberFor<Block>,
-		block: NumberFor<Block>,
-	) -> ClientResult<Option<Block::Hash>> {
-		self.read_cht_root(CHANGES_TRIE_CHT_PREFIX, cht_size, block)
-	}
-
 	fn finalize_header(&self, id: BlockId<Block>) -> ClientResult<()> {
 		if let Some(header) = self.header(id)? {
 			let mut transaction = Transaction::new();
@@ -612,6 +596,26 @@ impl<Block> Storage<Block> for LightStorage<Block>
 	}
 }
 
+impl<Block> ChtRootStorage<Block> for LightStorage<Block>
+	where Block: BlockT,
+{
+	fn header_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> ClientResult<Option<Block::Hash>> {
+		self.read_cht_root(HEADER_CHT_PREFIX, cht_size, block)
+	}
+
+	fn changes_trie_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> ClientResult<Option<Block::Hash>> {
+		self.read_cht_root(CHANGES_TRIE_CHT_PREFIX, cht_size, block)
+	}
+}
+
 /// Build the key for inserting header-CHT at given block.
 fn cht_key<N: TryInto<u32>>(cht_type: u8, block: N) -> ClientResult<[u8; 5]> {
 	let mut key = [cht_type; 5];

substrate/client/service/src/chain_ops/build_spec.rs

@@ -0,0 +1,82 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use sp_runtime::traits::{Block as BlockT, NumberFor, Saturating, One};
+use sp_blockchain::HeaderBackend;
+use crate::{TFullBackend, TLightBackend};
+use std::sync::Arc;
+use sp_runtime::generic::BlockId;
+
+/// An error for if this function is being called on a full node.
+pub const CHT_ROOT_ERROR: &str =
+	"Backend doesn't store CHT roots. Make sure you're calling this on a light client.";
+
+/// Something that might allow access to a `ChtRootStorage`.
+pub trait MaybeChtRootStorageProvider<Block> {
+	/// Potentially get a reference to a `ChtRootStorage`.
+	fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage<Block>>;
+}
+
+impl<Block: BlockT> MaybeChtRootStorageProvider<Block> for TFullBackend<Block> {
+	fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage<Block>> {
+		None
+	}
+}
+
+impl<Block: BlockT> MaybeChtRootStorageProvider<Block> for TLightBackend<Block> {
+	fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage<Block>> {
+		Some(self.blockchain().storage())
+	}
+}
+
+/// Build a `LightSyncState` from the CHT roots stored in a backend.
+pub fn build_light_sync_state<TBl, TCl, TBackend>(
+	client: Arc<TCl>,
+	backend: Arc<TBackend>,
+) -> Result<sc_chain_spec::LightSyncState<TBl>, sp_blockchain::Error>
+	where
+		TBl: BlockT,
+		TCl: HeaderBackend<TBl>,
+		TBackend: MaybeChtRootStorageProvider<TBl>,
+{
+	let storage = backend.cht_root_storage().ok_or(CHT_ROOT_ERROR)?;
+
+	let finalized_hash = client.info().finalized_hash;
+	let finalized_number = client.info().finalized_number;
+
+	use sc_client_api::cht;
+
+	let mut chts = Vec::new();
+
+	// We can't fetch a CHT root later than `finalized_number - 2 * cht_size`.
+	let cht_size_x_2 = cht::size::<NumberFor::<TBl>>() * NumberFor::<TBl>::from(2);
+
+	let mut number = NumberFor::<TBl>::one();
+
+	while number <= finalized_number.saturating_sub(cht_size_x_2) {
+		match storage.header_cht_root(cht::size(), number)? {
+			Some(cht_root) => chts.push(cht_root),
+			None => log::error!("No CHT found for block {}", number),
+		}
+
+		number += cht::size();
+	}
+
+	Ok(sc_chain_spec::LightSyncState {
+		header: client.header(BlockId::Hash(finalized_hash))?.unwrap(),
+		chts,
+	})
+}

substrate/client/service/src/chain_ops/mod.rs

@@ -21,9 +21,11 @@ mod export_blocks;
 mod export_raw_state;
 mod import_blocks;
 mod revert_chain;
+mod build_spec;
 
 pub use check_block::*;
 pub use export_blocks::*;
 pub use export_raw_state::*;
 pub use import_blocks::*;
 pub use revert_chain::*;
+pub use build_spec::*;

substrate/client/service/test/src/client/light.rs

@@ -42,7 +42,7 @@ use sc_executor::{NativeExecutor, WasmExecutionMethod, RuntimeVersion, NativeVer
 use sp_core::{H256, NativeOrEncoded, testing::TaskExecutor};
 use sc_client_api::{
 	blockchain::Info, backend::NewBlockState, Backend as ClientBackend, ProofProvider,
-	in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain},
+	in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain}, ChtRootStorage,
 	AuxStore, Storage, CallExecutor, cht, ExecutionStrategy, StorageProof, BlockImportOperation,
 	RemoteCallRequest, StorageProvider, ChangesProof, RemoteBodyRequest, RemoteReadRequest,
 	RemoteChangesRequest, FetchChecker, RemoteReadChildRequest, RemoteHeaderRequest, BlockBackend,
@@ -164,6 +164,16 @@ impl Storage<Block> for DummyStorage {
 		Err(ClientError::Backend("Test error".into()))
 	}
 
+	fn cache(&self) -> Option<Arc<dyn BlockchainCache<Block>>> {
+		None
+	}
+
+	fn usage_info(&self) -> Option<sc_client_api::UsageInfo> {
+		None
+	}
+}
+
+impl ChtRootStorage<Block> for DummyStorage {
 	fn header_cht_root(&self, _cht_size: u64, _block: u64) -> ClientResult<Option<Hash>> {
 		Err(ClientError::Backend("Test error".into()))
 	}
@@ -177,14 +187,6 @@ impl Storage<Block> for DummyStorage {
 			).into())
 			.map(Some)
 	}
-
-	fn cache(&self) -> Option<Arc<dyn BlockchainCache<Block>>> {
-		None
-	}
-
-	fn usage_info(&self) -> Option<sc_client_api::UsageInfo> {
-		None
-	}
 }
 
 struct DummyCallExecutor;