Dynamically generate CHT roots on a full client (#6944)

* Generate CHT roots on a full client

* add changes_trie_root function

* Add a test

* Line widths

* Fix sc-service-test

* Clarify comments

* Revert comments

substrate/client/api/src/backend.rs

@@ -536,3 +536,21 @@ pub fn changes_tries_state_at_block<'a, Block: BlockT>(
 		None => Ok(None),
 	}
 }
+
+/// Provide CHT roots. These are stored on a light client and generated dynamically on a full
+/// client.
+pub trait ProvideChtRoots<Block: BlockT> {
+	/// Get headers CHT root for given block. Returns None if the block is not a part of any CHT.
+	fn header_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> sp_blockchain::Result<Option<Block::Hash>>;
+
+	/// Get changes trie CHT root for given block. Returns None if the block is not a part of any CHT.
+	fn changes_trie_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> sp_blockchain::Result<Option<Block::Hash>>;
+}

substrate/client/api/src/in_mem.rs

@@ -35,7 +35,7 @@ use sp_state_machine::{
 use sp_blockchain::{CachedHeaderMetadata, HeaderMetadata};
 
 use crate::{
-	backend::{self, NewBlockState},
+	backend::{self, NewBlockState, ProvideChtRoots},
 	blockchain::{
 		self, BlockStatus, HeaderBackend, well_known_cache_keys::Id as CacheKeyId
 	},
@@ -456,7 +456,7 @@ impl<Block: BlockT> light::Storage<Block> for Blockchain<Block>
 	}
 }
 
-impl<Block: BlockT> light::ChtRootStorage<Block> for Blockchain<Block> {
+impl<Block: BlockT> ProvideChtRoots<Block> for Blockchain<Block> {
 	fn header_cht_root(
 		&self,
 		_cht_size: NumberFor<Block>,

substrate/client/api/src/light.rs

@@ -32,7 +32,7 @@ use sp_blockchain::{
 	HeaderMetadata, well_known_cache_keys, HeaderBackend, Cache as BlockchainCache,
 	Error as ClientError, Result as ClientResult,
 };
-use crate::{backend::{AuxStore, NewBlockState}, UsageInfo};
+use crate::{backend::{AuxStore, NewBlockState}, UsageInfo, ProvideChtRoots};
 
 /// Remote call request.
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
@@ -233,7 +233,7 @@ pub trait FetchChecker<Block: BlockT>: Send + Sync {
 
 /// Light client blockchain storage.
 pub trait Storage<Block: BlockT>: AuxStore + HeaderBackend<Block>
-	+ HeaderMetadata<Block, Error=ClientError> + ChtRootStorage<Block>
+	+ HeaderMetadata<Block, Error=ClientError> + ProvideChtRoots<Block>
 {
 	/// Store new header. Should refuse to revert any finalized blocks.
 	///
@@ -263,23 +263,6 @@ pub trait Storage<Block: BlockT>: AuxStore + HeaderBackend<Block>
 	fn usage_info(&self) -> Option<UsageInfo>;
 }
 
-/// Light client CHT root storage.
-pub trait ChtRootStorage<Block: BlockT> {
-	/// Get headers CHT root for given block. Returns None if the block is not pruned (not a part of any CHT).
-	fn header_cht_root(
-		&self,
-		cht_size: NumberFor<Block>,
-		block: NumberFor<Block>,
-	) -> ClientResult<Option<Block::Hash>>;
-
-	/// Get changes trie CHT root for given block. Returns None if the block is not pruned (not a part of any CHT).
-	fn changes_trie_cht_root(
-		&self,
-		cht_size: NumberFor<Block>,
-		block: NumberFor<Block>,
-	) -> ClientResult<Option<Block::Hash>>;
-}
-
 /// Remote header.
 #[derive(Debug)]
 pub enum LocalOrRemote<Data, Request> {

substrate/client/db/src/lib.rs

@@ -54,8 +54,8 @@ use std::collections::{HashMap, HashSet};
 
 use sc_client_api::{
 	UsageInfo, MemoryInfo, IoInfo, MemorySize,
-	backend::{NewBlockState, PrunableStateChangesTrieStorage},
-	leaves::{LeafSet, FinalizationDisplaced},
+	backend::{NewBlockState, PrunableStateChangesTrieStorage, ProvideChtRoots},
+	leaves::{LeafSet, FinalizationDisplaced}, cht,
 };
 use sp_blockchain::{
 	Result as ClientResult, Error as ClientError,
@@ -70,7 +70,7 @@ use sp_core::ChangesTrieConfiguration;
 use sp_core::offchain::storage::{OffchainOverlayedChange, OffchainOverlayedChanges};
 use sp_core::storage::{well_known_keys, ChildInfo};
 use sp_arithmetic::traits::Saturating;
-use sp_runtime::{generic::BlockId, Justification, Storage};
+use sp_runtime::{generic::{DigestItem, BlockId}, Justification, Storage};
 use sp_runtime::traits::{
 	Block as BlockT, Header as HeaderT, NumberFor, Zero, One, SaturatedConversion, HashFor,
 };
@@ -405,6 +405,14 @@ impl<Block: BlockT> BlockchainDb<Block> {
 			meta.finalized_hash = hash;
 		}
 	}
+
+	// Get block changes trie root, if available.
+	fn changes_trie_root(&self, block: BlockId<Block>) -> ClientResult<Option<Block::Hash>> {
+		self.header(block)
+			.map(|header| header.and_then(|header|
+				header.digest().log(DigestItem::as_changes_trie_root)
+					.cloned()))
+	}
 }
 
 impl<Block: BlockT> sc_client_api::blockchain::HeaderBackend<Block> for BlockchainDb<Block> {
@@ -525,6 +533,58 @@ impl<Block: BlockT> HeaderMetadata<Block> for BlockchainDb<Block> {
 	}
 }
 
+impl<Block: BlockT> ProvideChtRoots<Block> for BlockchainDb<Block> {
+	fn header_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> sp_blockchain::Result<Option<Block::Hash>> {
+		let cht_number = match cht::block_to_cht_number(cht_size, block) {
+			Some(number) => number,
+			None => return Ok(None),
+		};
+
+		let cht_start: NumberFor<Block> = cht::start_number(cht::size(), cht_number);
+
+		let mut current_num = cht_start;
+		let cht_range = ::std::iter::from_fn(|| {
+			let old_current_num = current_num;
+			current_num = current_num + One::one();
+			Some(old_current_num)
+		});
+
+		cht::compute_root::<Block::Header, HashFor<Block>, _>(
+			cht::size(), cht_number, cht_range.map(|num| self.hash(num))
+		).map(Some)
+	}
+
+	fn changes_trie_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> sp_blockchain::Result<Option<Block::Hash>> {
+		let cht_number = match cht::block_to_cht_number(cht_size, block) {
+			Some(number) => number,
+			None => return Ok(None),
+		};
+
+		let cht_start: NumberFor<Block> = cht::start_number(cht::size(), cht_number);
+
+		let mut current_num = cht_start;
+		let cht_range = ::std::iter::from_fn(|| {
+			let old_current_num = current_num;
+			current_num = current_num + One::one();
+			Some(old_current_num)
+		});
+
+		cht::compute_root::<Block::Header, HashFor<Block>, _>(
+			cht::size(),
+			cht_number,
+			cht_range.map(|num| self.changes_trie_root(BlockId::Number(num))),
+		).map(Some)
+	}
+}
+
 /// Database transaction
 pub struct BlockImportOperation<Block: BlockT> {
 	old_state: SyncingCachingState<RefTrackingState<Block>, Block>,
@@ -2329,4 +2389,29 @@ pub(crate) mod tests {
 			backend.commit_operation(op).unwrap_err();
 		}
 	}
+
+	#[test]
+	fn header_cht_root_works() {
+		use sc_client_api::ProvideChtRoots;
+
+		let backend = Backend::<Block>::new_test(10, 10);
+
+		// insert 1 + SIZE + SIZE + 1 blocks so that CHT#0 is created
+		let mut prev_hash = insert_header(&backend, 0, Default::default(), None, Default::default());
+		let cht_size: u64 = cht::size();
+		for i in 1..1 + cht_size + cht_size + 1 {
+			prev_hash = insert_header(&backend, i, prev_hash, None, Default::default());
+		}
+
+		let blockchain = backend.blockchain();
+
+		let cht_root_1 = blockchain.header_cht_root(cht_size, cht::start_number(cht_size, 0))
+			.unwrap().unwrap();
+		let cht_root_2 = blockchain.header_cht_root(cht_size, cht::start_number(cht_size, 0) + cht_size / 2)
+			.unwrap().unwrap();
+		let cht_root_3 = blockchain.header_cht_root(cht_size, cht::end_number(cht_size, 0))
+			.unwrap().unwrap();
+		assert_eq!(cht_root_1, cht_root_2);
+		assert_eq!(cht_root_2, cht_root_3);
+	}
 }

substrate/client/db/src/light.rs

@@ -23,11 +23,11 @@ use std::convert::TryInto;
 use parking_lot::RwLock;
 
 use sc_client_api::{
-	cht, backend::{AuxStore, NewBlockState}, UsageInfo,
+	cht, backend::{AuxStore, NewBlockState, ProvideChtRoots}, UsageInfo,
 	blockchain::{
 		BlockStatus, Cache as BlockchainCache, Info as BlockchainInfo,
 	},
-	Storage, ChtRootStorage,
+	Storage,
 };
 use sp_blockchain::{
 	CachedHeaderMetadata, HeaderMetadata, HeaderMetadataCache,
@@ -596,7 +596,7 @@ impl<Block> Storage<Block> for LightStorage<Block>
 	}
 }
 
-impl<Block> ChtRootStorage<Block> for LightStorage<Block>
+impl<Block> ProvideChtRoots<Block> for LightStorage<Block>
 	where Block: BlockT,
 {
 	fn header_cht_root(

substrate/client/light/src/blockchain.rs

@@ -29,7 +29,7 @@ use sp_blockchain::{
 };
 pub use sc_client_api::{
 	backend::{
-		AuxStore, NewBlockState
+		AuxStore, NewBlockState, ProvideChtRoots,
 	},
 	blockchain::{
 		Backend as BlockchainBackend, BlockStatus, Cache as BlockchainCache,
@@ -173,3 +173,21 @@ impl<S, Block: BlockT> RemoteBlockchain<Block> for Blockchain<S>
 		}))
 	}
 }
+
+impl<S: Storage<Block>, Block: BlockT> ProvideChtRoots<Block> for Blockchain<S> {
+	fn header_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> sp_blockchain::Result<Option<Block::Hash>> {
+		self.storage().header_cht_root(cht_size, block)
+	}
+
+	fn changes_trie_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> sp_blockchain::Result<Option<Block::Hash>> {
+		self.storage().changes_trie_cht_root(cht_size, block)
+	}
+}

substrate/client/service/src/chain_ops/build_spec.rs

@@ -16,31 +16,9 @@
 
 use sp_runtime::traits::{Block as BlockT, NumberFor, Saturating, One};
 use sp_blockchain::HeaderBackend;
-use crate::{TFullBackend, TLightBackend};
 use std::sync::Arc;
 use sp_runtime::generic::BlockId;
-
-/// An error for if this function is being called on a full node.
-pub const CHT_ROOT_ERROR: &str =
-	"Backend doesn't store CHT roots. Make sure you're calling this on a light client.";
-
-/// Something that might allow access to a `ChtRootStorage`.
-pub trait MaybeChtRootStorageProvider<Block> {
-	/// Potentially get a reference to a `ChtRootStorage`.
-	fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage<Block>>;
-}
-
-impl<Block: BlockT> MaybeChtRootStorageProvider<Block> for TFullBackend<Block> {
-	fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage<Block>> {
-		None
-	}
-}
-
-impl<Block: BlockT> MaybeChtRootStorageProvider<Block> for TLightBackend<Block> {
-	fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage<Block>> {
-		Some(self.blockchain().storage())
-	}
-}
+use sc_client_api::ProvideChtRoots;
 
 /// Build a `LightSyncState` from the CHT roots stored in a backend.
 pub fn build_light_sync_state<TBl, TCl, TBackend>(
@@ -50,9 +28,10 @@ pub fn build_light_sync_state<TBl, TCl, TBackend>(
 	where
 		TBl: BlockT,
 		TCl: HeaderBackend<TBl>,
-		TBackend: MaybeChtRootStorageProvider<TBl>,
+		TBackend: sc_client_api::Backend<TBl>,
+		<TBackend as sc_client_api::Backend<TBl>>::Blockchain: ProvideChtRoots<TBl>,
 {
-	let storage = backend.cht_root_storage().ok_or(CHT_ROOT_ERROR)?;
+	let cht_root_provider = backend.blockchain();
 
 	let finalized_hash = client.info().finalized_hash;
 	let finalized_number = client.info().finalized_number;
@@ -67,7 +46,7 @@ pub fn build_light_sync_state<TBl, TCl, TBackend>(
 	let mut number = NumberFor::<TBl>::one();
 
 	while number <= finalized_number.saturating_sub(cht_size_x_2) {
-		match storage.header_cht_root(cht::size(), number)? {
+		match cht_root_provider.header_cht_root(cht::size(), number)? {
 			Some(cht_root) => chts.push(cht_root),
 			None => log::error!("No CHT found for block {}", number),
 		}

substrate/client/service/test/src/client/light.rs

@@ -42,7 +42,7 @@ use sc_executor::{NativeExecutor, WasmExecutionMethod, RuntimeVersion, NativeVer
 use sp_core::{H256, NativeOrEncoded, testing::TaskExecutor};
 use sc_client_api::{
 	blockchain::Info, backend::NewBlockState, Backend as ClientBackend, ProofProvider,
-	in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain}, ChtRootStorage,
+	in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain}, ProvideChtRoots,
 	AuxStore, Storage, CallExecutor, cht, ExecutionStrategy, StorageProof, BlockImportOperation,
 	RemoteCallRequest, StorageProvider, ChangesProof, RemoteBodyRequest, RemoteReadRequest,
 	RemoteChangesRequest, FetchChecker, RemoteReadChildRequest, RemoteHeaderRequest, BlockBackend,
@@ -173,7 +173,7 @@ impl Storage<Block> for DummyStorage {
 	}
 }
 
-impl ChtRootStorage<Block> for DummyStorage {
+impl ProvideChtRoots<Block> for DummyStorage {
 	fn header_cht_root(&self, _cht_size: u64, _block: u64) -> ClientResult<Option<Hash>> {
 		Err(ClientError::Backend("Test error".into()))
 	}