Remove request multiplexer (#3624)

* WIP: Get rid of request multiplexer.

* WIP

* Receiver for handling of incoming requests.

* Get rid of useless `Fault` abstraction.

The things the type system let us do are not worth getting abstracted in
its own type. Instead error handling is going to be merely a pattern.

* Make most things compile again.

* Port availability distribution away from request multiplexer.

* Formatting.

* Port dispute distribution over.

* Fixup statement distribution.

* Handle request directly in collator protocol.

+ Only allow fatal errors at top level.

* Use direct request channel for availability recovery.

* Finally get rid of request multiplexer

Fixes #2842 and paves the way for more back pressure possibilities.

* Fix overseer and statement distribution tests.

* Fix collator protocol and network bridge tests.

* Fix tests in availability recovery.

* Fix availability distribution tests.

* Fix dispute distribution tests.

* Add missing dependency

* Typos.

* Review remarks.

* More remarks.

polkadot/node/network/availability-recovery/src/lib.rs

@@ -26,6 +26,7 @@ use std::{
 use futures::{
 	channel::oneshot,
 	future::{BoxFuture, FutureExt, RemoteHandle},
+	pin_mut,
 	prelude::*,
 	stream::FuturesUnordered,
 	task::{Context, Poll},
@@ -36,9 +37,10 @@ use rand::seq::SliceRandom;
 use polkadot_erasure_coding::{branch_hash, branches, obtain_chunks_v1, recovery_threshold};
 use polkadot_node_network_protocol::{
 	request_response::{
-		self as req_res, request::RequestError, OutgoingRequest, Recipient, Requests,
+		self as req_res, incoming, outgoing::RequestError, v1 as request_v1,
+		IncomingRequestReceiver, OutgoingRequest, Recipient, Requests,
 	},
-	IfDisconnected,
+	IfDisconnected, UnifiedReputationChange as Rep,
 };
 use polkadot_node_primitives::{AvailableData, ErasureChunk};
 use polkadot_node_subsystem_util::request_session_info;
@@ -68,9 +70,13 @@ const N_PARALLEL: usize = 50;
 // Size of the LRU cache where we keep recovered data.
 const LRU_SIZE: usize = 16;
 
+const COST_INVALID_REQUEST: Rep = Rep::CostMajor("Peer sent unparsable request");
+
 /// The Availability Recovery Subsystem.
 pub struct AvailabilityRecoverySubsystem {
 	fast_path: bool,
+	/// Receiver for available data requests.
+	req_receiver: IncomingRequestReceiver<request_v1::AvailableDataFetchingRequest>,
 }
 
 struct RequestFromBackersPhase {
@@ -750,13 +756,17 @@ where
 
 impl AvailabilityRecoverySubsystem {
 	/// Create a new instance of `AvailabilityRecoverySubsystem` which starts with a fast path to request data from backers.
-	pub fn with_fast_path() -> Self {
-		Self { fast_path: true }
+	pub fn with_fast_path(
+		req_receiver: IncomingRequestReceiver<request_v1::AvailableDataFetchingRequest>,
+	) -> Self {
+		Self { fast_path: true, req_receiver }
 	}
 
 	/// Create a new instance of `AvailabilityRecoverySubsystem` which requests only chunks
-	pub fn with_chunks_only() -> Self {
-		Self { fast_path: false }
+	pub fn with_chunks_only(
+		req_receiver: IncomingRequestReceiver<request_v1::AvailableDataFetchingRequest>,
+	) -> Self {
+		Self { fast_path: false, req_receiver }
 	}
 
 	async fn run<Context>(self, mut ctx: Context) -> SubsystemResult<()>
@@ -765,8 +775,11 @@ impl AvailabilityRecoverySubsystem {
 		Context: overseer::SubsystemContext<Message = AvailabilityRecoveryMessage>,
 	{
 		let mut state = State::default();
+		let Self { fast_path, mut req_receiver } = self;
 
 		loop {
+			let recv_req = req_receiver.recv(|| vec![COST_INVALID_REQUEST]).fuse();
+			pin_mut!(recv_req);
 			futures::select! {
 				v = ctx.recv().fuse() => {
 					match v? {
@@ -789,7 +802,7 @@ impl AvailabilityRecoverySubsystem {
 										&mut ctx,
 										receipt,
 										session_index,
-										maybe_backing_group.filter(|_| self.fast_path),
+										maybe_backing_group.filter(|_| fast_path),
 										response_sender,
 									).await {
 										tracing::warn!(
@@ -799,24 +812,37 @@ impl AvailabilityRecoverySubsystem {
 										);
 									}
 								}
-								AvailabilityRecoveryMessage::AvailableDataFetchingRequest(req) => {
-									match query_full_data(&mut ctx, req.payload.candidate_hash).await {
-										Ok(res) => {
-											let _ = req.send_response(res.into());
-										}
-										Err(e) => {
-											tracing::debug!(
-												target: LOG_TARGET,
-												err = ?e,
-												"Failed to query available data.",
-											);
+							}
+						}
+					}
+				}
+				in_req = recv_req => {
+					match in_req {
+						Ok(req) => {
+							match query_full_data(&mut ctx, req.payload.candidate_hash).await {
+								Ok(res) => {
+									let _ = req.send_response(res.into());
+								}
+								Err(e) => {
+									tracing::debug!(
+										target: LOG_TARGET,
+										err = ?e,
+										"Failed to query available data.",
+									);
 
-											let _ = req.send_response(None.into());
-										}
-									}
+									let _ = req.send_response(None.into());
 								}
 							}
 						}
+						Err(incoming::Error::Fatal(f)) => return Err(SubsystemError::with_origin("availability-recovery", f)),
+						Err(incoming::Error::NonFatal(err)) => {
+							tracing::debug!(
+								target: LOG_TARGET,
+								?err,
+								"Decoding incoming request failed"
+							);
+							continue
+						}
 					}
 				}
 				output = state.interactions.select_next_some() => {

polkadot/node/network/availability-recovery/src/tests.rs

@@ -21,9 +21,12 @@ use futures::{executor, future};
 use futures_timer::Delay;
 
 use parity_scale_codec::Encode;
+use polkadot_node_network_protocol::request_response::IncomingRequest;
 
 use super::*;
 
+use sc_network::config::RequestResponseConfig;
+
 use polkadot_erasure_coding::{branches, obtain_chunks_v1 as obtain_chunks};
 use polkadot_node_primitives::{BlockData, PoV};
 use polkadot_node_subsystem_util::TimeoutExt;
@@ -37,8 +40,8 @@ use polkadot_subsystem_testhelpers as test_helpers;
 
 type VirtualOverseer = test_helpers::TestSubsystemContextHandle<AvailabilityRecoveryMessage>;
 
-fn test_harness_fast_path<T: Future<Output = VirtualOverseer>>(
-	test: impl FnOnce(VirtualOverseer) -> T,
+fn test_harness_fast_path<T: Future<Output = (VirtualOverseer, RequestResponseConfig)>>(
+	test: impl FnOnce(VirtualOverseer, RequestResponseConfig) -> T,
 ) {
 	let _ = env_logger::builder()
 		.is_test(true)
@@ -49,27 +52,29 @@ fn test_harness_fast_path<T: Future<Output = VirtualOverseer>>(
 
 	let (context, virtual_overseer) = test_helpers::make_subsystem_context(pool.clone());
 
-	let subsystem = AvailabilityRecoverySubsystem::with_fast_path();
-	let subsystem = subsystem.run(context);
+	let (collation_req_receiver, req_cfg) = IncomingRequest::get_config_receiver();
+	let subsystem = AvailabilityRecoverySubsystem::with_fast_path(collation_req_receiver);
+	let subsystem = async {
+		subsystem.run(context).await.unwrap();
+	};
 
-	let test_fut = test(virtual_overseer);
+	let test_fut = test(virtual_overseer, req_cfg);
 
 	futures::pin_mut!(test_fut);
 	futures::pin_mut!(subsystem);
 
 	executor::block_on(future::join(
 		async move {
-			let mut overseer = test_fut.await;
+			let (mut overseer, _req_cfg) = test_fut.await;
 			overseer_signal(&mut overseer, OverseerSignal::Conclude).await;
 		},
 		subsystem,
 	))
 	.1
-	.unwrap();
 }
 
-fn test_harness_chunks_only<T: Future<Output = VirtualOverseer>>(
-	test: impl FnOnce(VirtualOverseer) -> T,
+fn test_harness_chunks_only<T: Future<Output = (VirtualOverseer, RequestResponseConfig)>>(
+	test: impl FnOnce(VirtualOverseer, RequestResponseConfig) -> T,
 ) {
 	let _ = env_logger::builder()
 		.is_test(true)
@@ -80,17 +85,18 @@ fn test_harness_chunks_only<T: Future<Output = VirtualOverseer>>(
 
 	let (context, virtual_overseer) = test_helpers::make_subsystem_context(pool.clone());
 
-	let subsystem = AvailabilityRecoverySubsystem::with_chunks_only();
+	let (collation_req_receiver, req_cfg) = IncomingRequest::get_config_receiver();
+	let subsystem = AvailabilityRecoverySubsystem::with_chunks_only(collation_req_receiver);
 	let subsystem = subsystem.run(context);
 
-	let test_fut = test(virtual_overseer);
+	let test_fut = test(virtual_overseer, req_cfg);
 
 	futures::pin_mut!(test_fut);
 	futures::pin_mut!(subsystem);
 
 	executor::block_on(future::join(
 		async move {
-			let mut overseer = test_fut.await;
+			let (mut overseer, _req_cfg) = test_fut.await;
 			overseer_signal(&mut overseer, OverseerSignal::Conclude).await;
 		},
 		subsystem,
@@ -432,7 +438,7 @@ impl Default for TestState {
 fn availability_is_recovered_from_chunks_if_no_group_provided() {
 	let test_state = TestState::default();
 
-	test_harness_fast_path(|mut virtual_overseer| async move {
+	test_harness_fast_path(|mut virtual_overseer, req_cfg| async move {
 		overseer_signal(
 			&mut virtual_overseer,
 			OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf {
@@ -510,7 +516,7 @@ fn availability_is_recovered_from_chunks_if_no_group_provided() {
 
 		// A request times out with `Unavailable` error.
 		assert_eq!(rx.await.unwrap().unwrap_err(), RecoveryError::Unavailable);
-		virtual_overseer
+		(virtual_overseer, req_cfg)
 	});
 }
 
@@ -518,7 +524,7 @@ fn availability_is_recovered_from_chunks_if_no_group_provided() {
 fn availability_is_recovered_from_chunks_even_if_backing_group_supplied_if_chunks_only() {
 	let test_state = TestState::default();
 
-	test_harness_chunks_only(|mut virtual_overseer| async move {
+	test_harness_chunks_only(|mut virtual_overseer, req_cfg| async move {
 		overseer_signal(
 			&mut virtual_overseer,
 			OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf {
@@ -596,7 +602,7 @@ fn availability_is_recovered_from_chunks_even_if_backing_group_supplied_if_chunk
 
 		// A request times out with `Unavailable` error.
 		assert_eq!(rx.await.unwrap().unwrap_err(), RecoveryError::Unavailable);
-		virtual_overseer
+		(virtual_overseer, req_cfg)
 	});
 }
 
@@ -604,7 +610,7 @@ fn availability_is_recovered_from_chunks_even_if_backing_group_supplied_if_chunk
 fn bad_merkle_path_leads_to_recovery_error() {
 	let mut test_state = TestState::default();
 
-	test_harness_fast_path(|mut virtual_overseer| async move {
+	test_harness_fast_path(|mut virtual_overseer, req_cfg| async move {
 		overseer_signal(
 			&mut virtual_overseer,
 			OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf {
@@ -654,7 +660,7 @@ fn bad_merkle_path_leads_to_recovery_error() {
 
 		// A request times out with `Unavailable` error.
 		assert_eq!(rx.await.unwrap().unwrap_err(), RecoveryError::Unavailable);
-		virtual_overseer
+		(virtual_overseer, req_cfg)
 	});
 }
 
@@ -662,7 +668,7 @@ fn bad_merkle_path_leads_to_recovery_error() {
 fn wrong_chunk_index_leads_to_recovery_error() {
 	let mut test_state = TestState::default();
 
-	test_harness_fast_path(|mut virtual_overseer| async move {
+	test_harness_fast_path(|mut virtual_overseer, req_cfg| async move {
 		overseer_signal(
 			&mut virtual_overseer,
 			OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf {
@@ -711,7 +717,7 @@ fn wrong_chunk_index_leads_to_recovery_error() {
 
 		// A request times out with `Unavailable` error as there are no good peers.
 		assert_eq!(rx.await.unwrap().unwrap_err(), RecoveryError::Unavailable);
-		virtual_overseer
+		(virtual_overseer, req_cfg)
 	});
 }
 
@@ -719,7 +725,7 @@ fn wrong_chunk_index_leads_to_recovery_error() {
 fn invalid_erasure_coding_leads_to_invalid_error() {
 	let mut test_state = TestState::default();
 
-	test_harness_fast_path(|mut virtual_overseer| async move {
+	test_harness_fast_path(|mut virtual_overseer, req_cfg| async move {
 		let pov = PoV { block_data: BlockData(vec![69; 64]) };
 
 		let (bad_chunks, bad_erasure_root) = derive_erasure_chunks_with_proofs_and_root(
@@ -776,7 +782,7 @@ fn invalid_erasure_coding_leads_to_invalid_error() {
 
 		// f+1 'valid' chunks can't produce correct data.
 		assert_eq!(rx.await.unwrap().unwrap_err(), RecoveryError::Invalid);
-		virtual_overseer
+		(virtual_overseer, req_cfg)
 	});
 }
 
@@ -784,7 +790,7 @@ fn invalid_erasure_coding_leads_to_invalid_error() {
 fn fast_path_backing_group_recovers() {
 	let test_state = TestState::default();
 
-	test_harness_fast_path(|mut virtual_overseer| async move {
+	test_harness_fast_path(|mut virtual_overseer, req_cfg| async move {
 		overseer_signal(
 			&mut virtual_overseer,
 			OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf {
@@ -826,7 +832,7 @@ fn fast_path_backing_group_recovers() {
 
 		// Recovered data should match the original one.
 		assert_eq!(rx.await.unwrap().unwrap(), test_state.available_data);
-		virtual_overseer
+		(virtual_overseer, req_cfg)
 	});
 }
 
@@ -834,7 +840,7 @@ fn fast_path_backing_group_recovers() {
 fn no_answers_in_fast_path_causes_chunk_requests() {
 	let test_state = TestState::default();
 
-	test_harness_fast_path(|mut virtual_overseer| async move {
+	test_harness_fast_path(|mut virtual_overseer, req_cfg| async move {
 		overseer_signal(
 			&mut virtual_overseer,
 			OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf {
@@ -888,7 +894,7 @@ fn no_answers_in_fast_path_causes_chunk_requests() {
 
 		// Recovered data should match the original one.
 		assert_eq!(rx.await.unwrap().unwrap(), test_state.available_data);
-		virtual_overseer
+		(virtual_overseer, req_cfg)
 	});
 }
 
@@ -896,7 +902,7 @@ fn no_answers_in_fast_path_causes_chunk_requests() {
 fn task_canceled_when_receivers_dropped() {
 	let test_state = TestState::default();
 
-	test_harness_chunks_only(|mut virtual_overseer| async move {
+	test_harness_chunks_only(|mut virtual_overseer, req_cfg| async move {
 		overseer_signal(
 			&mut virtual_overseer,
 			OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf {
@@ -925,7 +931,7 @@ fn task_canceled_when_receivers_dropped() {
 
 		for _ in 0..test_state.validators.len() {
 			match virtual_overseer.recv().timeout(TIMEOUT).await {
-				None => return virtual_overseer,
+				None => return (virtual_overseer, req_cfg),
 				Some(_) => continue,
 			}
 		}
@@ -938,7 +944,7 @@ fn task_canceled_when_receivers_dropped() {
 fn chunks_retry_until_all_nodes_respond() {
 	let test_state = TestState::default();
 
-	test_harness_chunks_only(|mut virtual_overseer| async move {
+	test_harness_chunks_only(|mut virtual_overseer, req_cfg| async move {
 		overseer_signal(
 			&mut virtual_overseer,
 			OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf {
@@ -992,7 +998,7 @@ fn chunks_retry_until_all_nodes_respond() {
 
 		// Recovered data should match the original one.
 		assert_eq!(rx.await.unwrap().unwrap_err(), RecoveryError::Unavailable);
-		virtual_overseer
+		(virtual_overseer, req_cfg)
 	});
 }
 
@@ -1000,7 +1006,7 @@ fn chunks_retry_until_all_nodes_respond() {
 fn returns_early_if_we_have_the_data() {
 	let test_state = TestState::default();
 
-	test_harness_chunks_only(|mut virtual_overseer| async move {
+	test_harness_chunks_only(|mut virtual_overseer, req_cfg| async move {
 		overseer_signal(
 			&mut virtual_overseer,
 			OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf {
@@ -1029,7 +1035,7 @@ fn returns_early_if_we_have_the_data() {
 		test_state.respond_to_available_data_query(&mut virtual_overseer, true).await;
 
 		assert_eq!(rx.await.unwrap().unwrap(), test_state.available_data);
-		virtual_overseer
+		(virtual_overseer, req_cfg)
 	});
 }
 
@@ -1037,7 +1043,7 @@ fn returns_early_if_we_have_the_data() {
 fn does_not_query_local_validator() {
 	let test_state = TestState::default();
 
-	test_harness_chunks_only(|mut virtual_overseer| async move {
+	test_harness_chunks_only(|mut virtual_overseer, req_cfg| async move {
 		overseer_signal(
 			&mut virtual_overseer,
 			OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf {
@@ -1088,6 +1094,6 @@ fn does_not_query_local_validator() {
 			.await;
 
 		assert_eq!(rx.await.unwrap().unwrap(), test_state.available_data);
-		virtual_overseer
+		(virtual_overseer, req_cfg)
 	});
 }