Prepare sc-network for ProtocolController/NotificationService (#14080)

* Prepare `sc-network` for `ProtocolController`/`NotificationService`

The upcoming notification protocol refactoring requires that protocols
are able to communicate with `sc-network` over unique and direct links.
This means that `sc-network` side of the link has to be created before
`sc-network` is initialized and that it is allowed to consume the object
as the receiver half of the link may not implement `Clone`.

Remove request-response and notification protocols from `NetworkConfiguration`
and create a new object that contains the configurations of these protocols
and which is consumable by `sc-network`. This is needed needed because, e.g.,
the receiver half of `NotificationService` is not clonable so `sc-network`
must consume it when it's initializing the protocols in `Notifications`.

Similar principe applies to `PeerStore`/`ProtocolController`: as per current
design, protocols are created before the network so `Protocol` cannot be
the one creating the `PeerStore` object. `FullNetworkConfiguration` will be
used to store the objects that `sc-network` will use to communicate with
protocols and it will also allow protocols to allocate handles so they
can directly communicate with `sc-network`.

* Fixes

* Update client/service/src/builder.rs

Co-authored-by: Dmitry Markin <dmitry@markin.tech>

* Updates

* Doc updates + cargo-fmt

---------

Co-authored-by: Dmitry Markin <dmitry@markin.tech>

substrate/client/network/test/src/lib.rs

@@ -50,8 +50,8 @@ use sc_consensus::{
 };
 use sc_network::{
 	config::{
-		MultiaddrWithPeerId, NetworkConfiguration, NonDefaultSetConfig, NonReservedPeerMode,
-		ProtocolId, Role, SyncMode, TransportConfig,
+		FullNetworkConfiguration, MultiaddrWithPeerId, NetworkConfiguration, NonDefaultSetConfig,
+		NonReservedPeerMode, ProtocolId, Role, SyncMode, TransportConfig,
 	},
 	request_responses::ProtocolConfig as RequestResponseConfig,
 	types::ProtocolName,
@@ -800,20 +800,6 @@ where
 		network_config.transport = TransportConfig::MemoryOnly;
 		network_config.listen_addresses = vec![listen_addr.clone()];
 		network_config.allow_non_globals_in_dht = true;
-		network_config
-			.request_response_protocols
-			.extend(config.request_response_protocols);
-		network_config.extra_sets = config
-			.notifications_protocols
-			.into_iter()
-			.map(|p| NonDefaultSetConfig {
-				notifications_protocol: p,
-				fallback_names: Vec::new(),
-				max_notification_size: 1024 * 1024,
-				handshake: None,
-				set_config: Default::default(),
-			})
-			.collect();
 		if let Some(connect_to) = config.connect_to_peers {
 			let addrs = connect_to
 				.iter()
@@ -826,6 +812,7 @@ where
 			network_config.default_peers_set.reserved_nodes = addrs;
 			network_config.default_peers_set.non_reserved_mode = NonReservedPeerMode::Deny;
 		}
+		let mut full_net_config = FullNetworkConfiguration::new(&network_config);
 
 		let protocol_id = ProtocolId::from("test-protocol-name");
 
@@ -890,7 +877,7 @@ where
 				Roles::from(if config.is_authority { &Role::Authority } else { &Role::Full }),
 				client.clone(),
 				None,
-				&network_config,
+				&full_net_config,
 				protocol_id.clone(),
 				&fork_id,
 				block_announce_validator,
@@ -906,6 +893,28 @@ where
 		let sync_service_import_queue = Box::new(sync_service.clone());
 		let sync_service = Arc::new(sync_service.clone());
 
+		for config in config.request_response_protocols {
+			full_net_config.add_request_response_protocol(config);
+		}
+		for config in [
+			block_request_protocol_config,
+			state_request_protocol_config,
+			light_client_request_protocol_config,
+			warp_protocol_config,
+		] {
+			full_net_config.add_request_response_protocol(config);
+		}
+
+		for protocol in config.notifications_protocols {
+			full_net_config.add_notification_protocol(NonDefaultSetConfig {
+				notifications_protocol: protocol,
+				fallback_names: Vec::new(),
+				max_notification_size: 1024 * 1024,
+				handshake: None,
+				set_config: Default::default(),
+			});
+		}
+
 		let genesis_hash =
 			client.hash(Zero::zero()).ok().flatten().expect("Genesis block exists; qed");
 		let network = NetworkWorker::new(sc_network::config::Params {
@@ -913,20 +922,13 @@ where
 			executor: Box::new(|f| {
 				tokio::spawn(f);
 			}),
-			network_config,
+			network_config: full_net_config,
 			genesis_hash,
 			protocol_id,
 			fork_id,
 			metrics_registry: None,
 			block_announce_config,
 			tx,
-			request_response_protocol_configs: [
-				block_request_protocol_config,
-				state_request_protocol_config,
-				light_client_request_protocol_config,
-				warp_protocol_config,
-			]
-			.to_vec(),
 		})
 		.unwrap();
 

substrate/client/network/test/src/service.rs

@@ -21,7 +21,7 @@ use libp2p::{Multiaddr, PeerId};
 
 use sc_consensus::{ImportQueue, Link};
 use sc_network::{
-	config::{self, MultiaddrWithPeerId, ProtocolId, TransportConfig},
+	config::{self, FullNetworkConfiguration, MultiaddrWithPeerId, ProtocolId, TransportConfig},
 	event::Event,
 	NetworkEventStream, NetworkNotification, NetworkPeers, NetworkService, NetworkStateInfo,
 	NetworkWorker,
@@ -77,6 +77,7 @@ struct TestNetworkBuilder {
 	listen_addresses: Vec<Multiaddr>,
 	set_config: Option<config::SetConfig>,
 	chain_sync_network: Option<(NetworkServiceProvider, NetworkServiceHandle)>,
+	notification_protocols: Vec<config::NonDefaultSetConfig>,
 	config: Option<config::NetworkConfiguration>,
 }
 
@@ -89,6 +90,7 @@ impl TestNetworkBuilder {
 			listen_addresses: Vec::new(),
 			set_config: None,
 			chain_sync_network: None,
+			notification_protocols: Vec::new(),
 			config: None,
 		}
 	}
@@ -98,6 +100,11 @@ impl TestNetworkBuilder {
 		self
 	}
 
+	pub fn with_notification_protocol(mut self, config: config::NonDefaultSetConfig) -> Self {
+		self.notification_protocols.push(config);
+		self
+	}
+
 	pub fn with_listen_addresses(mut self, addresses: Vec<Multiaddr>) -> Self {
 		self.listen_addresses = addresses;
 		self
@@ -115,13 +122,6 @@ impl TestNetworkBuilder {
 		);
 
 		let network_config = self.config.unwrap_or(config::NetworkConfiguration {
-			extra_sets: vec![config::NonDefaultSetConfig {
-				notifications_protocol: PROTOCOL_NAME.into(),
-				fallback_names: Vec::new(),
-				max_notification_size: 1024 * 1024,
-				handshake: None,
-				set_config: self.set_config.unwrap_or_default(),
-			}],
 			listen_addresses: self.listen_addresses,
 			transport: TransportConfig::MemoryOnly,
 			..config::NetworkConfiguration::new_local()
@@ -153,6 +153,7 @@ impl TestNetworkBuilder {
 
 		let protocol_id = ProtocolId::from("test-protocol-name");
 		let fork_id = Some(String::from("test-fork-id"));
+		let mut full_net_config = FullNetworkConfiguration::new(&network_config);
 
 		let block_request_protocol_config = {
 			let (handler, protocol_config) =
@@ -178,12 +179,11 @@ impl TestNetworkBuilder {
 		let (chain_sync_network_provider, chain_sync_network_handle) =
 			self.chain_sync_network.unwrap_or(NetworkServiceProvider::new());
 		let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000);
-
 		let (engine, chain_sync_service, block_announce_config) = SyncingEngine::new(
 			Roles::from(&config::Role::Full),
 			client.clone(),
 			None,
-			&network_config,
+			&full_net_config,
 			protocol_id.clone(),
 			&None,
 			Box::new(sp_consensus::block_validation::DefaultBlockAnnounceValidator),
@@ -197,6 +197,29 @@ impl TestNetworkBuilder {
 		)
 		.unwrap();
 		let mut link = self.link.unwrap_or(Box::new(chain_sync_service.clone()));
+
+		if !self.notification_protocols.is_empty() {
+			for config in self.notification_protocols {
+				full_net_config.add_notification_protocol(config);
+			}
+		} else {
+			full_net_config.add_notification_protocol(config::NonDefaultSetConfig {
+				notifications_protocol: PROTOCOL_NAME.into(),
+				fallback_names: Vec::new(),
+				max_notification_size: 1024 * 1024,
+				handshake: None,
+				set_config: self.set_config.unwrap_or_default(),
+			});
+		}
+
+		for config in [
+			block_request_protocol_config,
+			state_request_protocol_config,
+			light_client_request_protocol_config,
+		] {
+			full_net_config.add_request_response_protocol(config);
+		}
+
 		let genesis_hash =
 			client.hash(Zero::zero()).ok().flatten().expect("Genesis block exists; qed");
 		let worker = NetworkWorker::<
@@ -209,16 +232,10 @@ impl TestNetworkBuilder {
 				tokio::spawn(f);
 			}),
 			genesis_hash,
-			network_config,
+			network_config: full_net_config,
 			protocol_id,
 			fork_id,
 			metrics_registry: None,
-			request_response_protocol_configs: [
-				block_request_protocol_config,
-				state_request_protocol_config,
-				light_client_request_protocol_config,
-			]
-			.to_vec(),
 			tx,
 		})
 		.unwrap();
@@ -553,14 +570,14 @@ async fn fallback_name_working() {
 
 	let listen_addr = config::build_multiaddr![Memory(rand::random::<u64>())];
 	let (node1, mut events_stream1) = TestNetworkBuilder::new()
+		.with_notification_protocol(config::NonDefaultSetConfig {
+			notifications_protocol: NEW_PROTOCOL_NAME.into(),
+			fallback_names: vec![PROTOCOL_NAME.into()],
+			max_notification_size: 1024 * 1024,
+			handshake: None,
+			set_config: Default::default(),
+		})
 		.with_config(config::NetworkConfiguration {
-			extra_sets: vec![config::NonDefaultSetConfig {
-				notifications_protocol: NEW_PROTOCOL_NAME.into(),
-				fallback_names: vec![PROTOCOL_NAME.into()],
-				max_notification_size: 1024 * 1024,
-				handshake: None,
-				set_config: Default::default(),
-			}],
 			listen_addresses: vec![listen_addr.clone()],
 			transport: TransportConfig::MemoryOnly,
 			..config::NetworkConfiguration::new_local()