Remove client.backend (#2960)

* generalize tree_root to remove client.backend dependency

* replace client.backend.blockchain.header with client.header

* move used_state_cache_size into client info

* Create intermediate Setup State. Fixes #1134

* remove client.backend from finality proof

* update node-template

* move memory backend into test helper mode

* move test helper into client

* starting the big refactor, remove unused functions

* apply_finality

* apply_finality

* replacing more .backend from environment with client directly

* remove .backend from grandpa by using traits

* remove .backend from babe

* remove .backend from tests where it is not needed

* remove .backend from tests

* fixing tests

* fixing tests

* fixing more tests

* fixing tests

* fix all forks test

* fix style

* fixing unnecessary allocation

* remove old test.

* fix service docs

* apply suggestion

* minor clean ups

* turns out the test-helper features actually is being used!

* fixing line length.

* fix line length

* minor cleaning

* Apply suggestions from code review

thanks, @Basti

Co-Authored-By: Bastian Köcher <bkchr@users.noreply.github.com>

* address grumbles

* simplify finalize block on client

* move block back into inner function

* Apply suggestions from code review

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

* use as.ref instead of match

* Update core/client/src/backend.rs

Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>

substrate/core/network/src/test/mod.rs

@@ -27,10 +27,13 @@ use std::sync::Arc;
 use crate::config::build_multiaddr;
 use log::trace;
 use crate::chain::FinalityProofProvider;
-use client::{self, ClientInfo, BlockchainEvents, BlockImportNotification, FinalityNotifications, FinalityNotification};
-use client::{in_mem::Backend as InMemoryBackend, error::Result as ClientResult};
+use client::{
+	self, ClientInfo, BlockchainEvents, BlockImportNotification, FinalityNotifications,
+	FinalityNotification, LongestChain
+};
+use client::error::Result as ClientResult;
 use client::block_builder::BlockBuilder;
-use client::backend::AuxStore;
+use client::backend::{AuxStore, Backend, Finalizer};
 use crate::config::Roles;
 use consensus::import_queue::BasicQueue;
 use consensus::import_queue::{
@@ -45,7 +48,7 @@ use crate::{NetworkWorker, NetworkService, config::ProtocolId};
 use crate::config::{NetworkConfiguration, TransportConfig, BoxFinalityProofRequestBuilder};
 use libp2p::PeerId;
 use parking_lot::Mutex;
-use primitives::{H256, Blake2Hasher};
+use primitives::H256;
 use crate::protocol::{Context, ProtocolConfig};
 use sr_primitives::generic::{BlockId, OpaqueDigestItemId};
 use sr_primitives::traits::{Block as BlockT, Header, NumberFor};
@@ -55,13 +58,14 @@ use crate::specialization::NetworkSpecialization;
 use test_client::{self, AccountKeyring};
 
 pub use test_client::runtime::{Block, Extrinsic, Hash, Transfer};
-pub use test_client::TestClient;
+pub use test_client::{TestClient, TestClientBuilder, TestClientBuilderExt};
 
 type AuthorityId = babe_primitives::AuthorityId;
 
 #[cfg(any(test, feature = "test-helpers"))]
 /// A Verifier that accepts all blocks and passes them on with the configured
 /// finality to be imported.
+#[derive(Clone)]
 pub struct PassThroughVerifier(pub bool);
 
 #[cfg(any(test, feature = "test-helpers"))]
@@ -131,66 +135,57 @@ pub type PeersLightClient =
 
 #[derive(Clone)]
 pub enum PeersClient {
-	Full(Arc<PeersFullClient>),
-	Light(Arc<PeersLightClient>),
+	Full(Arc<PeersFullClient>, Arc<test_client::Backend>),
+	Light(Arc<PeersLightClient>, Arc<test_client::LightBackend>),
 }
 
 impl PeersClient {
 	pub fn as_full(&self) -> Option<Arc<PeersFullClient>> {
 		match *self {
-			PeersClient::Full(ref client) => Some(client.clone()),
+			PeersClient::Full(ref client, ref _backend) => Some(client.clone()),
 			_ => None,
 		}
 	}
 
 	pub fn as_block_import(&self) -> BoxBlockImport<Block> {
 		match *self {
-			PeersClient::Full(ref client) => Box::new(client.clone()) as _,
-			PeersClient::Light(ref client) => Box::new(client.clone()) as _,
-		}
-	}
-
-	pub fn as_in_memory_backend(&self) -> InMemoryBackend<Block, Blake2Hasher> {
-		#[allow(deprecated)]
-		match *self {
-			PeersClient::Full(ref client) => client.backend().as_in_memory(),
-			PeersClient::Light(_) => unimplemented!("TODO"),
+			PeersClient::Full(ref client, ref _backend) => Box::new(client.clone()) as _,
+			PeersClient::Light(ref client, ref _backend) => Box::new(client.clone()) as _,
 		}
 	}
 
 	pub fn get_aux(&self, key: &[u8]) -> ClientResult<Option<Vec<u8>>> {
-		#[allow(deprecated)]
 		match *self {
-			PeersClient::Full(ref client) => client.backend().get_aux(key),
-			PeersClient::Light(ref client) => client.backend().get_aux(key),
+			PeersClient::Full(ref client, ref _backend) => client.get_aux(key),
+			PeersClient::Light(ref client, ref _backend) => client.get_aux(key),
 		}
 	}
 
 	pub fn info(&self) -> ClientInfo<Block> {
 		match *self {
-			PeersClient::Full(ref client) => client.info(),
-			PeersClient::Light(ref client) => client.info(),
+			PeersClient::Full(ref client, ref _backend) => client.info(),
+			PeersClient::Light(ref client, ref _backend) => client.info(),
 		}
 	}
 
 	pub fn header(&self, block: &BlockId<Block>) -> ClientResult<Option<<Block as BlockT>::Header>> {
 		match *self {
-			PeersClient::Full(ref client) => client.header(block),
-			PeersClient::Light(ref client) => client.header(block),
+			PeersClient::Full(ref client, ref _backend) => client.header(block),
+			PeersClient::Light(ref client, ref _backend) => client.header(block),
 		}
 	}
 
 	pub fn justification(&self, block: &BlockId<Block>) -> ClientResult<Option<Justification>> {
 		match *self {
-			PeersClient::Full(ref client) => client.justification(block),
-			PeersClient::Light(ref client) => client.justification(block),
+			PeersClient::Full(ref client, ref _backend) => client.justification(block),
+			PeersClient::Light(ref client, ref _backend) => client.justification(block),
 		}
 	}
 
 	pub fn finality_notification_stream(&self) -> FinalityNotifications<Block> {
 		match *self {
-			PeersClient::Full(ref client) => client.finality_notification_stream(),
-			PeersClient::Light(ref client) => client.finality_notification_stream(),
+			PeersClient::Full(ref client, ref _backend) => client.finality_notification_stream(),
+			PeersClient::Light(ref client, ref _backend) => client.finality_notification_stream(),
 		}
 	}
 
@@ -201,8 +196,8 @@ impl PeersClient {
 		notify: bool
 	) -> ClientResult<()> {
 		match *self {
-			PeersClient::Full(ref client) => client.finalize_block(id, justification, notify),
-			PeersClient::Light(ref client) => client.finalize_block(id, justification, notify),
+			PeersClient::Full(ref client, ref _backend) => client.finalize_block(id, justification, notify),
+			PeersClient::Light(ref client, ref _backend) => client.finalize_block(id, justification, notify),
 		}
 	}
 }
@@ -216,6 +211,8 @@ pub struct Peer<D, S: NetworkSpecialization<Block>> {
 	/// We keep a copy of the block_import so that we can invoke it for locally-generated blocks,
 	/// instead of going through the import queue.
 	block_import: Box<dyn BlockImport<Block, Error = ConsensusError>>,
+	select_chain: Option<LongestChain<test_client::Backend, Block>>,
+	backend: Option<Arc<test_client::Backend>>,
 	network: NetworkWorker<Block, S, <Block as BlockT>::Hash>,
 	imported_blocks_stream: Box<dyn Stream<Item = BlockImportNotification<Block>, Error = ()> + Send>,
 	finality_notification_stream: Box<dyn Stream<Item = FinalityNotification<Block>, Error = ()> + Send>,
@@ -227,6 +224,11 @@ impl<D, S: NetworkSpecialization<Block>> Peer<D, S> {
 		self.network.service().is_major_syncing()
 	}
 
+	// Returns a clone of the local SelectChain, only available on full nodes
+	pub fn select_chain(&self) -> Option<LongestChain<test_client::Backend, Block>> {
+		self.select_chain.clone()
+	}
+
 	/// Returns the number of peers we're connected to.
 	pub fn num_peers(&self) -> usize {
 		self.network.num_connected_peers()
@@ -342,6 +344,33 @@ impl<D, S: NetworkSpecialization<Block>> Peer<D, S> {
 	pub fn network_service(&self) -> &Arc<NetworkService<Block, S, <Block as BlockT>::Hash>> {
 		&self.network.service()
 	}
+
+	/// Test helper to compare the blockchain state of multiple (networked)
+	/// clients.
+	/// Potentially costly, as it creates in-memory copies of both blockchains in order
+	/// to compare them. If you have easier/softer checks that are sufficient, e.g. 
+	/// by using .info(), you should probably use it instead of this.
+	pub fn blockchain_canon_equals(&self, other: &Self) -> bool {
+		if let (Some(mine), Some(others)) = (self.backend.clone(), other.backend.clone()) {
+			mine.as_in_memory().blockchain()
+				.canon_equals_to(others.as_in_memory().blockchain())
+		} else {
+			false
+		}
+	}
+
+	/// Count the current number of known blocks. Note that:
+	///  1. this might be expensive as it creates an in-memory-copy of the chain
+	///     to count the blocks, thus if you have a different way of testing this
+	///     (e.g. `info.best_hash`) - use that.
+	///  2. This is not always increasing nor accurate, as the
+	///     orphaned and proven-to-never-finalized blocks may be pruned at any time.
+	///     Therefore, this number can drop again.
+	pub fn blocks_count(&self) -> usize {
+		self.backend.as_ref().map(
+			|backend| backend.as_in_memory().blockchain().blocks_count()
+		).unwrap_or(0)
+	}
 }
 
 pub struct EmptyTransactionPool;
@@ -467,11 +496,14 @@ pub trait TestNetFactory: Sized {
 
 	/// Add a full peer.
 	fn add_full_peer(&mut self, config: &ProtocolConfig) {
-		let client = Arc::new(test_client::new());
-		let verifier = self.make_verifier(PeersClient::Full(client.clone()), config);
+		let test_client_builder = TestClientBuilder::with_default_backend();
+		let backend = test_client_builder.backend();
+		let (c, longest_chain) = test_client_builder.build_with_longest_chain();
+		let client = Arc::new(c);
+		let verifier = self.make_verifier(PeersClient::Full(client.clone(), backend.clone()), config);
 		let verifier = VerifierAdapter(Arc::new(Mutex::new(Box::new(verifier) as Box<_>)));
 		let (block_import, justification_import, finality_proof_import, finality_proof_request_builder, data)
-			= self.make_block_import(PeersClient::Full(client.clone()));
+			= self.make_block_import(PeersClient::Full(client.clone(), backend.clone()));
 		let block_import = BlockImportAdapter(Arc::new(Mutex::new(block_import)));
 
 		let import_queue = Box::new(BasicQueue::new(
@@ -491,7 +523,7 @@ pub trait TestNetFactory: Sized {
 				..NetworkConfiguration::default()
 			},
 			chain: client.clone(),
-			finality_proof_provider: self.make_finality_proof_provider(PeersClient::Full(client.clone())),
+			finality_proof_provider: self.make_finality_proof_provider(PeersClient::Full(client.clone(), backend.clone())),
 			finality_proof_request_builder,
 			on_demand: None,
 			transaction_pool: Arc::new(EmptyTransactionPool),
@@ -512,7 +544,9 @@ pub trait TestNetFactory: Sized {
 
 			peers.push(Peer {
 				data,
-				client: PeersClient::Full(client),
+				client: PeersClient::Full(client, backend.clone()),
+				select_chain: Some(longest_chain),
+				backend: Some(backend),
 				imported_blocks_stream,
 				finality_notification_stream,
 				block_import: Box::new(block_import),
@@ -527,11 +561,12 @@ pub trait TestNetFactory: Sized {
 		let mut config = config.clone();
 		config.roles = Roles::LIGHT;
 
-		let client = Arc::new(test_client::new_light());
-		let verifier = self.make_verifier(PeersClient::Light(client.clone()), &config);
+		let (c, backend) = test_client::new_light();
+		let client = Arc::new(c);
+		let verifier = self.make_verifier(PeersClient::Light(client.clone(), backend.clone()), &config);
 		let verifier = VerifierAdapter(Arc::new(Mutex::new(Box::new(verifier) as Box<_>)));
 		let (block_import, justification_import, finality_proof_import, finality_proof_request_builder, data)
-			= self.make_block_import(PeersClient::Light(client.clone()));
+			= self.make_block_import(PeersClient::Light(client.clone(), backend.clone()));
 		let block_import = BlockImportAdapter(Arc::new(Mutex::new(block_import)));
 
 		let import_queue = Box::new(BasicQueue::new(
@@ -551,7 +586,7 @@ pub trait TestNetFactory: Sized {
 				..NetworkConfiguration::default()
 			},
 			chain: client.clone(),
-			finality_proof_provider: self.make_finality_proof_provider(PeersClient::Light(client.clone())),
+			finality_proof_provider: self.make_finality_proof_provider(PeersClient::Light(client.clone(), backend.clone())),
 			finality_proof_request_builder,
 			on_demand: None,
 			transaction_pool: Arc::new(EmptyTransactionPool),
@@ -573,8 +608,10 @@ pub trait TestNetFactory: Sized {
 			peers.push(Peer {
 				data,
 				verifier,
+				select_chain: None,
+				backend: None,
 				block_import: Box::new(block_import),
-				client: PeersClient::Light(client),
+				client: PeersClient::Light(client, backend),
 				imported_blocks_stream,
 				finality_notification_stream,
 				network,

substrate/core/network/src/test/sync.rs

@@ -14,7 +14,6 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
-use client::{backend::Backend, blockchain::HeaderBackend};
 use crate::config::Roles;
 use consensus::BlockOrigin;
 use futures03::TryFutureExt as _;
@@ -36,8 +35,8 @@ fn test_ancestor_search_when_common_is(n: usize) {
 	net.peer(2).push_blocks(100, false);
 
 	net.block_until_sync(&mut runtime);
-	assert!(net.peer(0).client.as_in_memory_backend().blockchain()
-		.canon_equals_to(net.peer(1).client.as_in_memory_backend().blockchain()));
+	let peer1 = &net.peers()[1];
+	assert!(net.peers()[0].blockchain_canon_equals(peer1));
 }
 
 #[test]
@@ -156,8 +155,8 @@ fn sync_from_two_peers_works() {
 	net.peer(1).push_blocks(100, false);
 	net.peer(2).push_blocks(100, false);
 	net.block_until_sync(&mut runtime);
-	assert!(net.peer(0).client.as_in_memory_backend().blockchain()
-		.equals_to(net.peer(1).client.as_in_memory_backend().blockchain()));
+	let peer1 = &net.peers()[1];
+	assert!(net.peers()[0].blockchain_canon_equals(peer1));
 	assert!(!net.peer(0).is_major_syncing());
 }
 
@@ -170,8 +169,8 @@ fn sync_from_two_peers_with_ancestry_search_works() {
 	net.peer(1).push_blocks(100, false);
 	net.peer(2).push_blocks(100, false);
 	net.block_until_sync(&mut runtime);
-	assert!(net.peer(0).client.as_in_memory_backend().blockchain()
-		.canon_equals_to(net.peer(1).client.as_in_memory_backend().blockchain()));
+	let peer1 = &net.peers()[1];
+	assert!(net.peers()[0].blockchain_canon_equals(peer1));
 }
 
 #[test]
@@ -185,8 +184,8 @@ fn ancestry_search_works_when_backoff_is_one() {
 	net.peer(2).push_blocks(2, false);
 
 	net.block_until_sync(&mut runtime);
-	assert!(net.peer(0).client.as_in_memory_backend().blockchain()
-		.canon_equals_to(net.peer(1).client.as_in_memory_backend().blockchain()));
+	let peer1 = &net.peers()[1];
+	assert!(net.peers()[0].blockchain_canon_equals(peer1));
 }
 
 #[test]
@@ -200,8 +199,8 @@ fn ancestry_search_works_when_ancestor_is_genesis() {
 	net.peer(2).push_blocks(100, false);
 
 	net.block_until_sync(&mut runtime);
-	assert!(net.peer(0).client.as_in_memory_backend().blockchain()
-		.canon_equals_to(net.peer(1).client.as_in_memory_backend().blockchain()));
+	let peer1 = &net.peers()[1];
+	assert!(net.peers()[0].blockchain_canon_equals(peer1));
 }
 
 #[test]
@@ -226,8 +225,8 @@ fn sync_long_chain_works() {
 	let mut net = TestNet::new(2);
 	net.peer(1).push_blocks(500, false);
 	net.block_until_sync(&mut runtime);
-	assert!(net.peer(0).client.as_in_memory_backend().blockchain()
-		.equals_to(net.peer(1).client.as_in_memory_backend().blockchain()));
+	let peer1 = &net.peers()[1];
+	assert!(net.peers()[0].blockchain_canon_equals(peer1));
 }
 
 #[test]
@@ -238,8 +237,8 @@ fn sync_no_common_longer_chain_fails() {
 	net.peer(0).push_blocks(20, true);
 	net.peer(1).push_blocks(20, false);
 	net.block_until_sync(&mut runtime);
-	assert!(!net.peer(0).client.as_in_memory_backend().blockchain()
-		.canon_equals_to(net.peer(1).client.as_in_memory_backend().blockchain()));
+	let peer1 = &net.peers()[1];
+	assert!(!net.peers()[0].blockchain_canon_equals(peer1));
 }
 
 #[test]
@@ -334,11 +333,11 @@ fn sync_after_fork_works() {
 	net.peer(2).push_blocks(1, false);
 
 	// peer 1 has the best chain
-	let peer1_chain = net.peer(1).client.as_in_memory_backend().blockchain().clone();
 	net.block_until_sync(&mut runtime);
-	assert!(net.peer(0).client.as_in_memory_backend().blockchain().canon_equals_to(&peer1_chain));
-	assert!(net.peer(1).client.as_in_memory_backend().blockchain().canon_equals_to(&peer1_chain));
-	assert!(net.peer(2).client.as_in_memory_backend().blockchain().canon_equals_to(&peer1_chain));
+	let peer1 = &net.peers()[1];
+	assert!(net.peers()[0].blockchain_canon_equals(peer1));
+	(net.peers()[1].blockchain_canon_equals(peer1));
+	(net.peers()[2].blockchain_canon_equals(peer1));
 }
 
 #[test]
@@ -354,8 +353,8 @@ fn syncs_all_forks() {
 
 	net.block_until_sync(&mut runtime);
 	// Check that all peers have all of the blocks.
-	assert_eq!(9, net.peer(0).client.as_in_memory_backend().blockchain().blocks_count());
-	assert_eq!(9, net.peer(1).client.as_in_memory_backend().blockchain().blocks_count());
+	assert_eq!(9, net.peer(0).blocks_count());
+	assert_eq!(9, net.peer(1).blocks_count());
 }
 
 #[test]
@@ -368,11 +367,11 @@ fn own_blocks_are_announced() {
 
 	net.block_until_sync(&mut runtime);
 
-	assert_eq!(net.peer(0).client.as_in_memory_backend().blockchain().info().best_number, 1);
-	assert_eq!(net.peer(1).client.as_in_memory_backend().blockchain().info().best_number, 1);
-	let peer0_chain = net.peer(0).client.as_in_memory_backend().blockchain().clone();
-	assert!(net.peer(1).client.as_in_memory_backend().blockchain().canon_equals_to(&peer0_chain));
-	assert!(net.peer(2).client.as_in_memory_backend().blockchain().canon_equals_to(&peer0_chain));
+	assert_eq!(net.peer(0).client.info().chain.best_number, 1);
+	assert_eq!(net.peer(1).client.info().chain.best_number, 1);
+	let peer0 = &net.peers()[0];
+	assert!(net.peers()[1].blockchain_canon_equals(peer0));
+	(net.peers()[2].blockchain_canon_equals(peer0));
 }
 
 #[test]