Use DB backend in test client (#1779)

* use DB backend in test client

* Update core/client/db/src/lib.rs

Co-Authored-By: svyatonik <svyatonik@gmail.com>

substrate/Cargo.lock

@@ -4134,9 +4134,11 @@ dependencies = [
 name = "substrate-test-client"
 version = "0.1.0"
 dependencies = [
+ "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-codec 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "sr-primitives 0.1.0",
  "substrate-client 0.1.0",
+ "substrate-client-db 0.1.0",
  "substrate-consensus-common 0.1.0",
  "substrate-executor 0.1.0",
  "substrate-keyring 0.1.0",

substrate/core/client/db/Cargo.toml

@@ -10,6 +10,7 @@ log = "0.4"
 kvdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
 # FIXME replace with release as soon as our rocksdb changes are released upstream https://github.com/paritytech/parity-common/issues/88
 kvdb-rocksdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
+kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d", optional = true }
 lru-cache = "0.1.1"
 hash-db = { version = "0.11" }
 primitives = { package = "substrate-primitives", path = "../../primitives" }
@@ -27,3 +28,7 @@ kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b031
 substrate-keyring = { path = "../../keyring" }
 test-client = { package = "substrate-test-client", path = "../../test-client" }
 env_logger = { version = "0.6" }
+
+[features]
+default = []
+test-helpers = ["kvdb-memorydb"]

substrate/core/client/db/src/lib.rs

@@ -57,6 +57,9 @@ use crate::storage_cache::{CachingState, SharedCache, new_shared_cache};
 use log::{trace, debug, warn};
 pub use state_db::PruningMode;
 
+#[cfg(feature = "test-helpers")]
+use client::in_mem::Backend as InMemoryBackend;
+
 const CANONICALIZATION_DELAY: u64 = 4096;
 const MIN_BLOCKS_TO_KEEP_CHANGES_TRIES_FOR: u64 = 32768;
 const STATE_CACHE_SIZE_BYTES: usize = 16 * 1024 * 1024;
@@ -463,8 +466,10 @@ impl<Block: BlockT> client::backend::PrunableStateChangesTrieStorage<Blake2Hashe
 
 impl<Block: BlockT> state_machine::ChangesTrieRootsStorage<Blake2Hasher> for DbChangesTrieStorage<Block> {
 	fn root(&self, anchor: &state_machine::ChangesTrieAnchorBlockId<H256>, block: u64) -> Result<Option<H256>, String> {
-		// check API requirement
-		assert!(block <= anchor.number, "API requirement");
+		// check API requirement: we can't get NEXT block(s) based on anchor
+		if block > anchor.number {
+			return Err(format!("Can't get changes trie root at {} using anchor at {}", block, anchor.number));
+		}
 
 		// we need to get hash of the block to resolve changes trie root
 		let block_id = if block <= self.meta.read().finalized_number.as_() {
@@ -531,8 +536,8 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 		Backend::from_kvdb(db as Arc<_>, config.pruning, canonicalization_delay)
 	}
 
-	#[cfg(test)]
-	fn new_test(keep_blocks: u32, canonicalization_delay: u64) -> Self {
+	#[cfg(any(test, feature = "test-helpers"))]
+	pub fn new_test(keep_blocks: u32, canonicalization_delay: u64) -> Self {
 		use utils::NUM_COLUMNS;
 
 		let db = Arc::new(::kvdb_memorydb::create(NUM_COLUMNS));
@@ -570,6 +575,54 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 		})
 	}
 
+	/// Returns in-memory blockchain that contains the same set of blocks that the self.
+	#[cfg(feature = "test-helpers")]
+	pub fn as_in_memory(&self) -> InMemoryBackend<Block, Blake2Hasher> {
+		use client::backend::{Backend as ClientBackend, BlockImportOperation};
+		use client::blockchain::Backend as BlockchainBackend;
+
+		let inmem = InMemoryBackend::<Block, Blake2Hasher>::new();
+
+		// get all headers hashes && sort them by number (could be duplicate)
+		let mut headers: Vec<(NumberFor<Block>, Block::Hash, Block::Header)> = Vec::new();
+		for (_, header) in self.blockchain.db.iter(columns::HEADER) {
+			let header = Block::Header::decode(&mut &header[..]).unwrap();
+			let hash = header.hash();
+			let number = *header.number();
+			let pos = headers.binary_search_by(|item| item.0.cmp(&number));
+			match pos {
+				Ok(pos) => headers.insert(pos, (number, hash, header)),
+				Err(pos) => headers.insert(pos, (number, hash, header)),
+			}
+		}
+
+		// insert all other headers + bodies + justifications
+		let info = self.blockchain.info().unwrap();
+		for (number, hash, header) in headers {
+			let id = BlockId::Hash(hash);
+			let justification = self.blockchain.justification(id).unwrap();
+			let body = self.blockchain.body(id).unwrap();
+			let state = self.state_at(id).unwrap().pairs();
+
+			let new_block_state = if number.is_zero() {
+				NewBlockState::Final
+			} else if hash == info.best_hash {
+				NewBlockState::Best
+			} else {
+				NewBlockState::Normal
+			};
+			let mut op = inmem.begin_operation().unwrap();
+			op.set_block_data(header, body, justification, new_block_state).unwrap();
+			op.update_db_storage(state.into_iter().map(|(k, v)| (None, k, Some(v))).collect()).unwrap();
+			inmem.commit_operation(op).unwrap();
+		}
+
+		// and now finalize the best block we have
+		inmem.finalize_block(BlockId::Hash(info.finalized_hash), None).unwrap();
+
+		inmem
+	}
+
 	/// Handle setting head within a transaction. `route_to` should be the last
 	/// block that existed in the database. `best_to` should be the best block
 	/// to be set.
@@ -712,8 +765,8 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 		operation.apply_aux(&mut transaction);
 
 		let mut meta_updates = Vec::new();
+		let mut last_finalized_hash = self.blockchain.meta.read().finalized_hash;
 		if !operation.finalized_blocks.is_empty() {
-			let mut last_finalized_hash = self.blockchain.meta.read().finalized_hash;
 			for (block, justification) in operation.finalized_blocks {
 				let block_hash = self.blockchain.expect_block_hash_from_id(&block)?;
 				let block_header = self.blockchain.expect_header(BlockId::Hash(block_hash))?;
@@ -787,7 +840,7 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 
 			if finalized {
 				// TODO: ensure best chain contains this block.
-				self.ensure_sequential_finalization(header, None)?;
+				self.ensure_sequential_finalization(header, Some(last_finalized_hash))?;
 				self.note_finalized(&mut transaction, header, hash)?;
 			} else {
 				// canonicalize blocks which are old enough, regardless of finality.

substrate/core/client/db/src/light.rs

@@ -72,8 +72,8 @@ impl<Block> LightStorage<Block>
 		Self::from_kvdb(db as Arc<_>)
 	}
 
-	#[cfg(test)]
-	pub(crate) fn new_test() -> Self {
+	#[cfg(any(test, feature = "test-helpers"))]
+	pub fn new_test() -> Self {
 		use utils::NUM_COLUMNS;
 
 		let db = Arc::new(::kvdb_memorydb::create(NUM_COLUMNS));
@@ -493,7 +493,7 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
 	}
 
 	fn cache(&self) -> Option<&BlockchainCache<Block>> {
-		None
+		Some(&self.cache)
 	}
 }
 

substrate/core/client/src/client.rs

@@ -1628,7 +1628,7 @@ pub(crate) mod tests {
 		client.import(BlockOrigin::Own, builder.bake().unwrap()).unwrap();
 
 		assert_eq!(client.info().unwrap().chain.best_number, 1);
-		assert!(client.state_at(&BlockId::Number(1)).unwrap() != client.state_at(&BlockId::Number(0)).unwrap());
+		assert!(client.state_at(&BlockId::Number(1)).unwrap().pairs() != client.state_at(&BlockId::Number(0)).unwrap().pairs());
 		assert_eq!(
 			client.runtime_api().balance_of(
 				&BlockId::Number(client.info().unwrap().chain.best_number),
@@ -1647,14 +1647,11 @@ pub(crate) mod tests {
 
 	#[test]
 	fn client_uses_authorities_from_blockchain_cache() {
-		let client = test_client::new();
-		test_client::client::in_mem::cache_authorities_at(
-			client.backend().blockchain(),
-			Default::default(),
-			Some(vec![[1u8; 32].into()]));
-		assert_eq!(client.authorities_at(
-			&BlockId::Hash(Default::default())).unwrap(),
-			vec![[1u8; 32].into()]);
+		let client = test_client::new_light();
+		let genesis_hash = client.header(&BlockId::Number(0)).unwrap().unwrap().hash();
+		// authorities cache is first filled in genesis block
+		// => should be read from cache here (remote request will fail in this test)
+		assert!(!client.authorities_at(&BlockId::Hash(genesis_hash)).unwrap().is_empty());
 	}
 
 	#[test]
@@ -1680,7 +1677,7 @@ pub(crate) mod tests {
 		client.import(BlockOrigin::Own, builder.bake().unwrap()).unwrap();
 
 		assert_eq!(client.info().unwrap().chain.best_number, 1);
-		assert!(client.state_at(&BlockId::Number(1)).unwrap() != client.state_at(&BlockId::Number(0)).unwrap());
+		assert!(client.state_at(&BlockId::Number(1)).unwrap().pairs() != client.state_at(&BlockId::Number(0)).unwrap().pairs());
 		assert_eq!(client.body(&BlockId::Number(1)).unwrap().unwrap().len(), 1)
 	}
 

substrate/core/network/src/test/sync.rs

@@ -31,7 +31,8 @@ fn sync_from_two_peers_works() {
 	net.peer(1).push_blocks(100, false);
 	net.peer(2).push_blocks(100, false);
 	net.sync();
-	assert!(net.peer(0).client.backend().blockchain().equals_to(net.peer(1).client.backend().blockchain()));
+	assert!(net.peer(0).client.backend().as_in_memory().blockchain()
+		.equals_to(net.peer(1).client.backend().as_in_memory().blockchain()));
 	let status = net.peer(0).status();
 	assert_eq!(status.sync.state, SyncState::Idle);
 }
@@ -45,7 +46,8 @@ fn sync_from_two_peers_with_ancestry_search_works() {
 	net.peer(2).push_blocks(100, false);
 	net.restart_peer(0);
 	net.sync();
-	assert!(net.peer(0).client.backend().blockchain().canon_equals_to(net.peer(1).client.backend().blockchain()));
+	assert!(net.peer(0).client.backend().as_in_memory().blockchain()
+		.canon_equals_to(net.peer(1).client.backend().as_in_memory().blockchain()));
 }
 
 #[test]
@@ -58,7 +60,8 @@ fn sync_long_chain_works() {
 	net.sync();
 	// Wait for peers to get up to speed.
 	thread::sleep(time::Duration::from_millis(1000));
-	assert!(net.peer(0).client.backend().blockchain().equals_to(net.peer(1).client.backend().blockchain()));
+	assert!(net.peer(0).client.backend().as_in_memory().blockchain()
+		.equals_to(net.peer(1).client.backend().as_in_memory().blockchain()));
 }
 
 #[test]
@@ -68,7 +71,8 @@ fn sync_no_common_longer_chain_fails() {
 	net.peer(0).push_blocks(20, true);
 	net.peer(1).push_blocks(20, false);
 	net.sync();
-	assert!(!net.peer(0).client.backend().blockchain().canon_equals_to(net.peer(1).client.backend().blockchain()));
+	assert!(!net.peer(0).client.backend().as_in_memory().blockchain()
+		.canon_equals_to(net.peer(1).client.backend().as_in_memory().blockchain()));
 }
 
 #[test]
@@ -111,11 +115,11 @@ fn sync_after_fork_works() {
 	net.peer(2).push_blocks(1, false);
 
 	// peer 1 has the best chain
-	let peer1_chain = net.peer(1).client.backend().blockchain().clone();
+	let peer1_chain = net.peer(1).client.backend().as_in_memory().blockchain().clone();
 	net.sync();
-	assert!(net.peer(0).client.backend().blockchain().canon_equals_to(&peer1_chain));
-	assert!(net.peer(1).client.backend().blockchain().canon_equals_to(&peer1_chain));
-	assert!(net.peer(2).client.backend().blockchain().canon_equals_to(&peer1_chain));
+	assert!(net.peer(0).client.backend().as_in_memory().blockchain().canon_equals_to(&peer1_chain));
+	assert!(net.peer(1).client.backend().as_in_memory().blockchain().canon_equals_to(&peer1_chain));
+	assert!(net.peer(2).client.backend().as_in_memory().blockchain().canon_equals_to(&peer1_chain));
 }
 
 #[test]
@@ -131,8 +135,8 @@ fn syncs_all_forks() {
 
 	net.sync();
 	// Check that all peers have all of the blocks.
-	assert_eq!(9, net.peer(0).client.backend().blockchain().blocks_count());
-	assert_eq!(9, net.peer(1).client.backend().blockchain().blocks_count());
+	assert_eq!(9, net.peer(0).client.backend().as_in_memory().blockchain().blocks_count());
+	assert_eq!(9, net.peer(1).client.backend().as_in_memory().blockchain().blocks_count());
 }
 
 #[test]
@@ -147,9 +151,9 @@ fn own_blocks_are_announced() {
 	net.sync();
 	assert_eq!(net.peer(0).client.backend().blockchain().info().unwrap().best_number, 1);
 	assert_eq!(net.peer(1).client.backend().blockchain().info().unwrap().best_number, 1);
-	let peer0_chain = net.peer(0).client.backend().blockchain().clone();
-	assert!(net.peer(1).client.backend().blockchain().canon_equals_to(&peer0_chain));
-	assert!(net.peer(2).client.backend().blockchain().canon_equals_to(&peer0_chain));
+	let peer0_chain = net.peer(0).client.backend().as_in_memory().blockchain().clone();
+	assert!(net.peer(1).client.backend().as_in_memory().blockchain().canon_equals_to(&peer0_chain));
+	assert!(net.peer(2).client.backend().as_in_memory().blockchain().canon_equals_to(&peer0_chain));
 }
 
 #[test]

substrate/core/state-machine/src/changes_trie/changes_iterator.rs

@@ -40,6 +40,9 @@ pub fn key_changes<'a, S: Storage<H>, H: Hasher>(
 	max: u64,
 	key: &'a [u8],
 ) -> Result<DrilldownIterator<'a, S, S, H>, String> where H::Out: HeapSizeOf {
+	// we can't query any roots before root
+	let max = ::std::cmp::min(max, end.number);
+
 	Ok(DrilldownIterator {
 		essence: DrilldownIteratorEssence {
 			key,
@@ -67,6 +70,9 @@ pub fn key_changes_proof<S: Storage<H>, H: Hasher>(
 	max: u64,
 	key: &[u8],
 ) -> Result<Vec<Vec<u8>>, String> where H::Out: HeapSizeOf {
+	// we can't query any roots before root
+	let max = ::std::cmp::min(max, end.number);
+
 	let mut iter = ProvingDrilldownIterator {
 		essence: DrilldownIteratorEssence {
 			key,
@@ -104,6 +110,9 @@ pub fn key_changes_proof_check<S: RootsStorage<H>, H: Hasher>(
 	max: u64,
 	key: &[u8]
 ) -> Result<Vec<(u64, u32)>, String> where H::Out: HeapSizeOf {
+	// we can't query any roots before root
+	let max = ::std::cmp::min(max, end.number);
+
 	let mut proof_db = MemoryDB::<H>::default();
 	for item in proof {
 		proof_db.insert(&item);

substrate/core/test-client/Cargo.toml

@@ -6,6 +6,8 @@ edition = "2018"
 
 [dependencies]
 client = { package = "substrate-client", path = "../client" }
+client-db = { package = "substrate-client-db", path = "../client/db", features = ["test-helpers"] }
+futures = { version = "0.1.17" }
 parity-codec = "3.0"
 executor = { package = "substrate-executor", path = "../executor" }
 consensus = { package = "substrate-consensus-common", path = "../consensus/common" }

substrate/core/test-client/src/lib.rs

@@ -34,9 +34,10 @@ pub use runtime;
 pub use consensus;
 
 use std::sync::Arc;
+use futures::future::FutureResult;
 use primitives::Blake2Hasher;
 use runtime_primitives::StorageOverlay;
-use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Hash as HashT};
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Hash as HashT, NumberFor};
 use runtime::genesismap::{GenesisConfig, additional_storage_with_genesis};
 use keyring::Keyring;
 use state_machine::ExecutionStrategy;
@@ -59,7 +60,7 @@ mod local_executor {
 pub use local_executor::LocalExecutor;
 
 /// Test client database backend.
-pub type Backend = client::in_mem::Backend<runtime::Block, Blake2Hasher>;
+pub type Backend = client_db::Backend<runtime::Block>;
 
 /// Test client executor.
 pub type Executor = client::LocalCallExecutor<
@@ -67,16 +68,60 @@ pub type Executor = client::LocalCallExecutor<
 	executor::NativeExecutor<LocalExecutor>,
 >;
 
+/// Test client light database backend.
+pub type LightBackend = client::light::backend::Backend<
+	client_db::light::LightStorage<runtime::Block>,
+	LightFetcher,
+	Blake2Hasher,
+>;
+
+/// Test client light fetcher.
+pub struct LightFetcher;
+
+/// Test client light executor.
+pub type LightExecutor = client::light::call_executor::RemoteOrLocalCallExecutor<
+	runtime::Block,
+	LightBackend,
+	client::light::call_executor::RemoteCallExecutor<
+		client::light::blockchain::Blockchain<
+			client_db::light::LightStorage<runtime::Block>,
+			LightFetcher
+		>,
+		LightFetcher
+	>,
+	client::LocalCallExecutor<
+		client::light::backend::Backend<
+			client_db::light::LightStorage<runtime::Block>,
+			LightFetcher,
+			Blake2Hasher
+		>,
+		executor::NativeExecutor<LocalExecutor>
+	>
+>;
+
 /// Creates new client instance used for tests.
 pub fn new() -> client::Client<Backend, Executor, runtime::Block, runtime::RuntimeApi> {
-	new_with_backend(Arc::new(Backend::new()), false)
+	new_with_backend(Arc::new(Backend::new_test(::std::u32::MAX, ::std::u64::MAX)), false)
+}
+
+/// Creates new light client instance used for tests.
+pub fn new_light() -> client::Client<LightBackend, LightExecutor, runtime::Block, runtime::RuntimeApi> {
+	let storage = client_db::light::LightStorage::new_test();
+	let blockchain = Arc::new(client::light::blockchain::Blockchain::new(storage));
+	let backend = Arc::new(LightBackend::new(blockchain.clone()));
+	let executor = NativeExecutor::new(None);
+	let fetcher = Arc::new(LightFetcher);
+	let remote_call_executor = client::light::call_executor::RemoteCallExecutor::new(blockchain.clone(), fetcher);
+	let local_call_executor = client::LocalCallExecutor::new(backend.clone(), executor);
+	let call_executor = LightExecutor::new(backend.clone(), remote_call_executor, local_call_executor);
+	client::Client::new(backend, call_executor, genesis_storage(false), Default::default()).unwrap()
 }
 
 /// Creates new client instance used for tests with the given api execution strategy.
 pub fn new_with_execution_strategy(
 	execution_strategy: ExecutionStrategy
 ) -> client::Client<Backend, Executor, runtime::Block, runtime::RuntimeApi> {
-	let backend = Arc::new(Backend::new());
+	let backend = Arc::new(Backend::new_test(::std::u32::MAX, ::std::u64::MAX));
 	let executor = NativeExecutor::new(None);
 	let executor = LocalCallExecutor::new(backend.clone(), executor);
 
@@ -99,7 +144,7 @@ pub fn new_with_execution_strategy(
 pub fn new_with_changes_trie()
 	-> client::Client<Backend, Executor, runtime::Block, runtime::RuntimeApi>
 {
-	new_with_backend(Arc::new(Backend::new()), true)
+	new_with_backend(Arc::new(Backend::new_test(::std::u32::MAX, ::std::u64::MAX)), true)
 }
 
 /// Creates new client instance used for tests with an explicitly provided backend.
@@ -133,3 +178,38 @@ fn genesis_storage(support_changes_trie: bool) -> StorageOverlay {
 	storage.extend(additional_storage_with_genesis(&block));
 	storage
 }
+
+impl<Block: BlockT> client::light::fetcher::Fetcher<Block> for LightFetcher {
+	type RemoteHeaderResult = FutureResult<Block::Header, client::error::Error>;
+	type RemoteReadResult = FutureResult<Option<Vec<u8>>, client::error::Error>;
+	type RemoteCallResult = FutureResult<Vec<u8>, client::error::Error>;
+	type RemoteChangesResult = FutureResult<Vec<(NumberFor<Block>, u32)>, client::error::Error>;
+
+	fn remote_header(
+		&self,
+		_request: client::light::fetcher::RemoteHeaderRequest<Block::Header>,
+	) -> Self::RemoteHeaderResult {
+		unimplemented!("not (yet) used in tests")
+	}
+
+	fn remote_read(
+		&self,
+		_request: client::light::fetcher::RemoteReadRequest<Block::Header>,
+	) -> Self::RemoteReadResult {
+		unimplemented!("not (yet) used in tests")
+	}
+
+	fn remote_call(
+		&self,
+		_request: client::light::fetcher::RemoteCallRequest<Block::Header>,
+	) -> Self::RemoteCallResult {
+		unimplemented!("not (yet) used in tests")
+	}
+
+	fn remote_changes(
+		&self,
+		_request: client::light::fetcher::RemoteChangesRequest<Block::Header>,
+	) -> Self::RemoteChangesResult {
+		unimplemented!("not (yet) used in tests")
+	}
+}