Rework the event system of sc-network (#1370)

This commit introduces a new concept called `NotificationService` which allows Polkadot protocols to communicate with the underlying notification protocol implementation directly, without routing events through `NetworkWorker`. This implies that each protocol has its own service which it uses to communicate with remote peers and that each `NotificationService` is unique with respect to the underlying notification protocol, meaning `NotificationService` for the transaction protocol can only be used to send and receive transaction-related notifications. The `NotificationService` concept introduces two additional benefits: * allow protocols to start using custom handshakes * allow protocols to accept/reject inbound peers Previously the validation of inbound connections was solely the responsibility of `ProtocolController`. This caused issues with light peers and `SyncingEngine` as `ProtocolController` would accept more peers than `SyncingEngine` could accept which caused peers to have differing views of their own states. `SyncingEngine` would reject excess peers but these rejections were not properly communicated to those peers causing them to assume that they were accepted. With `NotificationService`, the local handshake is not sent to remote peer if peer is rejected which allows it to detect that it was rejected. This commit also deprecates the use of `NetworkEventStream` for all notification-related events and going forward only DHT events are provided through `NetworkEventStream`. If protocols wish to follow each other's events, they must introduce additional abtractions, as is done for GRANDPA and transactions protocols by following the syncing protocol through `SyncEventStream`. Fixes https://github.com/paritytech/polkadot-sdk/issues/512 Fixes https://github.com/paritytech/polkadot-sdk/issues/514 Fixes https://github.com/paritytech/polkadot-sdk/issues/515 Fixes https://github.com/paritytech/polkadot-sdk/issues/554 Fixes https://github.com/paritytech/polkadot-sdk/issues/556 --- These changes are transferred from https://github.com/paritytech/substrate/pull/14197 but there are no functional changes compared to that PR --------- Co-authored-by: Dmitry Markin <dmitry@markin.tech> Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>
2026-06-20 23:21:02 +00:00 · 2023-11-28 20:18:52 +02:00
parent ec3a61ed86
commit e71c484d5b
102 changed files with 5694 additions and 2603 deletions
@@ -51,7 +51,8 @@ use {
 	},
 	polkadot_node_core_dispute_coordinator::Config as DisputeCoordinatorConfig,
 	polkadot_node_network_protocol::{
-		peer_set::PeerSetProtocolNames, request_response::ReqProtocolNames,
+		peer_set::{PeerSet, PeerSetProtocolNames},
+		request_response::ReqProtocolNames,
 	},
 	sc_client_api::BlockBackend,
 	sc_transaction_pool_api::OffchainTransactionPoolFactory,
@@ -74,7 +75,7 @@ pub use {
 #[cfg(feature = "full-node")]
 use polkadot_node_subsystem::jaeger;

-use std::{path::PathBuf, sync::Arc, time::Duration};
+use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration};

 use prometheus_endpoint::Registry;
 #[cfg(feature = "full-node")]
@@ -809,9 +810,9 @@ pub fn new_full<OverseerGenerator: OverseerGen>(
 	// anything in terms of behaviour, but makes the logs more consistent with the other
 	// Substrate nodes.
 	let grandpa_protocol_name = grandpa::protocol_standard_name(&genesis_hash, &config.chain_spec);
-	net_config.add_notification_protocol(grandpa::grandpa_peers_set_config(
-		grandpa_protocol_name.clone(),
-	));
+	let (grandpa_protocol_config, grandpa_notification_service) =
+		grandpa::grandpa_peers_set_config(grandpa_protocol_name.clone());
+	net_config.add_notification_protocol(grandpa_protocol_config);

 	let beefy_gossip_proto_name =
 		beefy::gossip_protocol_name(&genesis_hash, config.chain_spec.fork_id());
@@ -824,12 +825,17 @@ pub fn new_full<OverseerGenerator: OverseerGen>(
 			client.clone(),
 			prometheus_registry.clone(),
 		);
-	if enable_beefy {
-		net_config.add_notification_protocol(beefy::communication::beefy_peers_set_config(
-			beefy_gossip_proto_name.clone(),
-		));
-		net_config.add_request_response_protocol(beefy_req_resp_cfg);
-	}
+	let beefy_notification_service = match enable_beefy {
+		false => None,
+		true => {
+			let (beefy_notification_config, beefy_notification_service) =
+				beefy::communication::beefy_peers_set_config(beefy_gossip_proto_name.clone());
+
+			net_config.add_notification_protocol(beefy_notification_config);
+			net_config.add_request_response_protocol(beefy_req_resp_cfg);
+			Some(beefy_notification_service)
+		},
+	};

 	// validation/collation protocols are enabled only if `Overseer` is enabled
 	let peerset_protocol_names =
@@ -840,13 +846,21 @@ pub fn new_full<OverseerGenerator: OverseerGen>(
 	//
 	// Collators and parachain full nodes require the collator and validator networking to send
 	// collations and to be able to recover PoVs.
-	if role.is_authority() || is_parachain_node.is_running_alongside_parachain_node() {
-		use polkadot_network_bridge::{peer_sets_info, IsAuthority};
-		let is_authority = if role.is_authority() { IsAuthority::Yes } else { IsAuthority::No };
-		for config in peer_sets_info(is_authority, &peerset_protocol_names) {
-			net_config.add_notification_protocol(config);
-		}
-	}
+	let notification_services =
+		if role.is_authority() || is_parachain_node.is_running_alongside_parachain_node() {
+			use polkadot_network_bridge::{peer_sets_info, IsAuthority};
+			let is_authority = if role.is_authority() { IsAuthority::Yes } else { IsAuthority::No };
+
+			peer_sets_info(is_authority, &peerset_protocol_names)
+				.into_iter()
+				.map(|(config, (peerset, service))| {
+					net_config.add_notification_protocol(config);
+					(peerset, service)
+				})
+				.collect::<HashMap<PeerSet, Box<dyn sc_network::NotificationService>>>()
+		} else {
+			std::collections::HashMap::new()
+		};

 	let req_protocol_names = ReqProtocolNames::new(&genesis_hash, config.chain_spec.fork_id());

@@ -1078,6 +1092,7 @@ pub fn new_full<OverseerGenerator: OverseerGen>(
 					offchain_transaction_pool_factory: OffchainTransactionPoolFactory::new(
 						transaction_pool.clone(),
 					),
+					notification_services,
 				},
 			)
 			.map_err(|e| {
@@ -1179,13 +1194,15 @@ pub fn new_full<OverseerGenerator: OverseerGen>(
 	// need a keystore, regardless of which protocol we use below.
 	let keystore_opt = if role.is_authority() { Some(keystore_container.keystore()) } else { None };

-	if enable_beefy {
+	// beefy is enabled if its notification service exists
+	if let Some(notification_service) = beefy_notification_service {
 		let justifications_protocol_name = beefy_on_demand_justifications_handler.protocol_name();
 		let network_params = beefy::BeefyNetworkParams {
 			network: network.clone(),
 			sync: sync_service.clone(),
 			gossip_protocol_name: beefy_gossip_proto_name,
 			justifications_protocol_name,
+			notification_service,
 			_phantom: core::marker::PhantomData::<Block>,
 		};
 		let payload_provider = beefy_primitives::mmr::MmrRootProvider::new(client.clone());
@@ -1265,6 +1282,7 @@ pub fn new_full<OverseerGenerator: OverseerGen>(
 			prometheus_registry: prometheus_registry.clone(),
 			shared_voter_state,
 			telemetry: telemetry.as_ref().map(|x| x.handle()),
+			notification_service: grandpa_notification_service,
 			offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(transaction_pool.clone()),
 		};

@@ -26,7 +26,7 @@ use polkadot_node_core_candidate_validation::Config as CandidateValidationConfig
 use polkadot_node_core_chain_selection::Config as ChainSelectionConfig;
 use polkadot_node_core_dispute_coordinator::Config as DisputeCoordinatorConfig;
 use polkadot_node_network_protocol::{
-	peer_set::PeerSetProtocolNames,
+	peer_set::{PeerSet, PeerSetProtocolNames},
 	request_response::{
 		v1 as request_v1, v2 as request_v2, IncomingRequestReceiver, ReqProtocolNames,
 	},
@@ -41,15 +41,16 @@ use polkadot_overseer::{
 	OverseerConnector, OverseerHandle, SpawnGlue,
 };

+use parking_lot::Mutex;
 use polkadot_primitives::runtime_api::ParachainHost;
 use sc_authority_discovery::Service as AuthorityDiscoveryService;
 use sc_client_api::AuxStore;
 use sc_keystore::LocalKeystore;
-use sc_network::NetworkStateInfo;
+use sc_network::{NetworkStateInfo, NotificationService};
 use sp_api::ProvideRuntimeApi;
 use sp_blockchain::HeaderBackend;
 use sp_consensus_babe::BabeApi;
-use std::sync::Arc;
+use std::{collections::HashMap, sync::Arc};

 pub use polkadot_approval_distribution::ApprovalDistribution as ApprovalDistributionSubsystem;
 pub use polkadot_availability_bitfield_distribution::BitfieldDistribution as BitfieldDistributionSubsystem;
@@ -140,6 +141,8 @@ where
 	pub peerset_protocol_names: PeerSetProtocolNames,
 	/// The offchain transaction pool factory.
 	pub offchain_transaction_pool_factory: OffchainTransactionPoolFactory<Block>,
+	/// Notification services for validation/collation protocols.
+	pub notification_services: HashMap<PeerSet, Box<dyn NotificationService>>,
 }

 /// Obtain a prepared `OverseerBuilder`, that is initialized
@@ -173,6 +176,7 @@ pub fn prepared_overseer_builder<Spawner, RuntimeClient>(
 		req_protocol_names,
 		peerset_protocol_names,
 		offchain_transaction_pool_factory,
+		notification_services,
 	}: OverseerGenArgs<Spawner, RuntimeClient>,
 ) -> Result<
 	InitializedOverseerBuilder<
@@ -218,6 +222,7 @@ where
 	use polkadot_node_subsystem_util::metrics::Metrics;

 	let metrics = <OverseerMetrics as MetricsTrait>::register(registry)?;
+	let notification_sinks = Arc::new(Mutex::new(HashMap::new()));

 	let spawner = SpawnGlue(spawner);

@@ -235,6 +240,7 @@ where
 			network_bridge_metrics.clone(),
 			req_protocol_names,
 			peerset_protocol_names.clone(),
+			notification_sinks.clone(),
 		))
 		.network_bridge_rx(NetworkBridgeRxSubsystem::new(
 			network_service.clone(),
@@ -242,6 +248,8 @@ where
 			Box::new(sync_service.clone()),
 			network_bridge_metrics,
 			peerset_protocol_names,
+			notification_services,
+			notification_sinks,
 		))
 		.availability_distribution(AvailabilityDistributionSubsystem::new(
 			keystore.clone(),