Network sync refactoring (part 4) (#11412)

* Remove direct dependency of `sc-network` on `sc-network-light`

* Move `WarpSyncProvider` trait and surrounding data structures into `sc-network-common`

* Move `WarpSyncProvider` trait and surrounding data structures into `sc-network-common`

* Create `sync` module in `sc-network-common`, create `ChainSync` trait there (not used yet), move a bunch of associated data structures from `sc-network-sync`

* Switch from concrete implementation to `ChainSync` trait from `sc-network-common`

* Introduce `OpaqueStateRequest`/`OpaqueStateResponse` to remove generics from `StateSync` trait

* Introduce `OpaqueBlockRequest`/`OpaqueBlockResponse`, make `scheme` module of `sc-network-sync` private

* Surface `sc-network-sync` into `sc-service` and make `sc-network` not depend on it anymore

* Remove now unnecessary dependency from `sc-network`

* Replace crate links with just text since dependencies are gone now

* Remove `warp_sync` re-export from `sc-network-common`

* Update copyright in network-related files

* Address review comments about documentation

* Apply review suggestion

* Rename `extra_requests` module to `metrics`

Co-authored-by: Bastian Köcher <info@kchr.de>

substrate/client/network/test/src/lib.rs

@@ -48,16 +48,21 @@ use sc_consensus::{
 };
 pub use sc_network::config::EmptyTransactionPool;
 use sc_network::{
-	block_request_handler::BlockRequestHandler,
 	config::{
-		MultiaddrWithPeerId, NetworkConfiguration, NonDefaultSetConfig, NonReservedPeerMode,
-		ProtocolConfig, Role, SyncMode, TransportConfig,
+		MultiaddrWithPeerId, NetworkConfiguration, NonDefaultSetConfig, NonReservedPeerMode, Role,
+		SyncMode, TransportConfig,
 	},
-	light_client_requests::handler::LightClientRequestHandler,
-	state_request_handler::StateRequestHandler,
-	warp_request_handler, Multiaddr, NetworkService, NetworkWorker,
+	Multiaddr, NetworkService, NetworkWorker,
 };
 pub use sc_network_common::config::ProtocolId;
+use sc_network_common::sync::warp::{
+	AuthorityList, EncodedProof, SetId, VerificationResult, WarpSyncProvider,
+};
+use sc_network_light::light_client_requests::handler::LightClientRequestHandler;
+use sc_network_sync::{
+	block_request_handler::BlockRequestHandler, state_request_handler::StateRequestHandler,
+	warp_request_handler, ChainSync,
+};
 use sc_service::client::Client;
 use sp_blockchain::{
 	well_known_cache_keys::{self, Id as CacheKeyId},
@@ -638,27 +643,26 @@ impl<B: BlockT> VerifierAdapter<B> {
 
 struct TestWarpSyncProvider<B: BlockT>(Arc<dyn HeaderBackend<B>>);
 
-impl<B: BlockT> warp_request_handler::WarpSyncProvider<B> for TestWarpSyncProvider<B> {
+impl<B: BlockT> WarpSyncProvider<B> for TestWarpSyncProvider<B> {
 	fn generate(
 		&self,
 		_start: B::Hash,
-	) -> Result<warp_request_handler::EncodedProof, Box<dyn std::error::Error + Send + Sync>> {
+	) -> Result<EncodedProof, Box<dyn std::error::Error + Send + Sync>> {
 		let info = self.0.info();
 		let best_header = self.0.header(BlockId::hash(info.best_hash)).unwrap().unwrap();
-		Ok(warp_request_handler::EncodedProof(best_header.encode()))
+		Ok(EncodedProof(best_header.encode()))
 	}
 	fn verify(
 		&self,
-		proof: &warp_request_handler::EncodedProof,
-		_set_id: warp_request_handler::SetId,
-		_authorities: warp_request_handler::AuthorityList,
-	) -> Result<warp_request_handler::VerificationResult<B>, Box<dyn std::error::Error + Send + Sync>>
-	{
-		let warp_request_handler::EncodedProof(encoded) = proof;
+		proof: &EncodedProof,
+		_set_id: SetId,
+		_authorities: AuthorityList,
+	) -> Result<VerificationResult<B>, Box<dyn std::error::Error + Send + Sync>> {
+		let EncodedProof(encoded) = proof;
 		let header = B::Header::decode(&mut encoded.as_slice()).unwrap();
-		Ok(warp_request_handler::VerificationResult::Complete(0, Default::default(), header))
+		Ok(VerificationResult::Complete(0, Default::default(), header))
 	}
-	fn current_authorities(&self) -> warp_request_handler::AuthorityList {
+	fn current_authorities(&self) -> AuthorityList {
 		Default::default()
 	}
 }
@@ -688,7 +692,7 @@ pub struct FullPeerConfig {
 	pub storage_chain: bool,
 }
 
-pub trait TestNetFactory: Sized
+pub trait TestNetFactory: Default + Sized
 where
 	<Self::BlockImport as BlockImport<Block>>::Transaction: Send,
 {
@@ -696,14 +700,8 @@ where
 	type BlockImport: BlockImport<Block, Error = ConsensusError> + Clone + Send + Sync + 'static;
 	type PeerData: Default;
 
-	/// These two need to be implemented!
-	fn from_config(config: &ProtocolConfig) -> Self;
-	fn make_verifier(
-		&self,
-		client: PeersClient,
-		config: &ProtocolConfig,
-		peer_data: &Self::PeerData,
-	) -> Self::Verifier;
+	/// This one needs to be implemented!
+	fn make_verifier(&self, client: PeersClient, peer_data: &Self::PeerData) -> Self::Verifier;
 
 	/// Get reference to peer.
 	fn peer(&mut self, i: usize) -> &mut Peer<Self::PeerData, Self::BlockImport>;
@@ -723,15 +721,10 @@ where
 		Self::PeerData,
 	);
 
-	fn default_config() -> ProtocolConfig {
-		ProtocolConfig::default()
-	}
-
 	/// Create new test network with this many peers.
 	fn new(n: usize) -> Self {
 		trace!(target: "test_network", "Creating test network");
-		let config = Self::default_config();
-		let mut net = Self::from_config(&config);
+		let mut net = Self::default();
 
 		for i in 0..n {
 			trace!(target: "test_network", "Adding peer {}", i);
@@ -767,11 +760,8 @@ where
 		let (block_import, justification_import, data) = self
 			.make_block_import(PeersClient { client: client.clone(), backend: backend.clone() });
 
-		let verifier = self.make_verifier(
-			PeersClient { client: client.clone(), backend: backend.clone() },
-			&Default::default(),
-			&data,
-		);
+		let verifier = self
+			.make_verifier(PeersClient { client: client.clone(), backend: backend.clone() }, &data);
 		let verifier = VerifierAdapter::new(verifier);
 
 		let import_queue = Box::new(BasicQueue::new(
@@ -845,6 +835,10 @@ where
 			protocol_config
 		};
 
+		let max_parallel_downloads = network_config.max_parallel_downloads;
+		let block_announce_validator = config
+			.block_announce_validator
+			.unwrap_or_else(|| Box::new(DefaultBlockAnnounceValidator));
 		let network = NetworkWorker::new(sc_network::config::Params {
 			role: if config.is_authority { Role::Authority } else { Role::Full },
 			executor: None,
@@ -856,9 +850,18 @@ where
 			transaction_pool: Arc::new(EmptyTransactionPool),
 			protocol_id,
 			import_queue,
-			block_announce_validator: config
-				.block_announce_validator
-				.unwrap_or_else(|| Box::new(DefaultBlockAnnounceValidator)),
+			create_chain_sync: Box::new(move |sync_mode, chain, warp_sync_provider| {
+				match ChainSync::new(
+					sync_mode,
+					chain,
+					block_announce_validator,
+					max_parallel_downloads,
+					warp_sync_provider,
+				) {
+					Ok(chain_sync) => Ok(Box::new(chain_sync)),
+					Err(error) => Err(Box::new(error).into()),
+				}
+			}),
 			metrics_registry: None,
 			block_request_protocol_config,
 			state_request_protocol_config,
@@ -1012,6 +1015,7 @@ where
 	}
 }
 
+#[derive(Default)]
 pub struct TestNet {
 	peers: Vec<Peer<(), PeersClient>>,
 }
@@ -1021,17 +1025,7 @@ impl TestNetFactory for TestNet {
 	type PeerData = ();
 	type BlockImport = PeersClient;
 
-	/// Create new test network with peers and given config.
-	fn from_config(_config: &ProtocolConfig) -> Self {
-		TestNet { peers: Vec::new() }
-	}
-
-	fn make_verifier(
-		&self,
-		_client: PeersClient,
-		_config: &ProtocolConfig,
-		_peer_data: &(),
-	) -> Self::Verifier {
+	fn make_verifier(&self, _client: PeersClient, _peer_data: &()) -> Self::Verifier {
 		PassThroughVerifier::new(false)
 	}
 
@@ -1081,6 +1075,7 @@ impl JustificationImport<Block> for ForceFinalized {
 	}
 }
 
+#[derive(Default)]
 pub struct JustificationTestNet(TestNet);
 
 impl TestNetFactory for JustificationTestNet {
@@ -1088,17 +1083,8 @@ impl TestNetFactory for JustificationTestNet {
 	type PeerData = ();
 	type BlockImport = PeersClient;
 
-	fn from_config(config: &ProtocolConfig) -> Self {
-		JustificationTestNet(TestNet::from_config(config))
-	}
-
-	fn make_verifier(
-		&self,
-		client: PeersClient,
-		config: &ProtocolConfig,
-		peer_data: &(),
-	) -> Self::Verifier {
-		self.0.make_verifier(client, config, peer_data)
+	fn make_verifier(&self, client: PeersClient, peer_data: &()) -> Self::Verifier {
+		self.0.make_verifier(client, peer_data)
 	}
 
 	fn peer(&mut self, i: usize) -> &mut Peer<Self::PeerData, Self::BlockImport> {