Remove request multiplexer (#3624)

* WIP: Get rid of request multiplexer.

* WIP

* Receiver for handling of incoming requests.

* Get rid of useless `Fault` abstraction.

The things the type system let us do are not worth getting abstracted in
its own type. Instead error handling is going to be merely a pattern.

* Make most things compile again.

* Port availability distribution away from request multiplexer.

* Formatting.

* Port dispute distribution over.

* Fixup statement distribution.

* Handle request directly in collator protocol.

+ Only allow fatal errors at top level.

* Use direct request channel for availability recovery.

* Finally get rid of request multiplexer

Fixes #2842 and paves the way for more back pressure possibilities.

* Fix overseer and statement distribution tests.

* Fix collator protocol and network bridge tests.

* Fix tests in availability recovery.

* Fix availability distribution tests.

* Fix dispute distribution tests.

* Add missing dependency

* Typos.

* Review remarks.

* More remarks.

polkadot/node/network/protocol/src/request_response/incoming/error.rs

@@ -0,0 +1,55 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Error handling related code and Error/Result definitions.
+
+use sc_network::PeerId;
+use thiserror::Error;
+
+use parity_scale_codec::Error as DecodingError;
+
+/// Errors that happen during reception/decoding of incoming requests.
+#[derive(Debug, Error, derive_more::From)]
+#[error(transparent)]
+pub enum Error {
+	/// All fatal errors.
+	Fatal(Fatal),
+	/// All nonfatal/potentially recoverable errors.
+	NonFatal(NonFatal),
+}
+
+/// Fatal errors when receiving incoming requests.
+#[derive(Debug, Error)]
+pub enum Fatal {
+	/// Incoming request stream exhausted. Should only happen on shutdown.
+	#[error("Incoming request channel got closed.")]
+	RequestChannelExhausted,
+}
+
+/// Non-fatal errors when receiving incoming requests.
+#[derive(Debug, Error)]
+pub enum NonFatal {
+	/// Decoding failed, we were able to change the peer's reputation accordingly.
+	#[error("Decoding request failed for peer {0}.")]
+	DecodingError(PeerId, #[source] DecodingError),
+
+	/// Decoding failed, but sending reputation change failed.
+	#[error("Decoding request failed for peer {0}, and changing reputation failed.")]
+	DecodingErrorNoReputationChange(PeerId, #[source] DecodingError),
+}
+
+/// General result based on above `Error`.
+pub type Result<T> = std::result::Result<T, Error>;

polkadot/node/network/protocol/src/request_response/incoming/mod.rs

@@ -0,0 +1,232 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::marker::PhantomData;
+
+use futures::{
+	channel::{mpsc, oneshot},
+	StreamExt,
+};
+
+use parity_scale_codec::{Decode, Encode};
+
+use sc_network::{config as netconfig, config::RequestResponseConfig, PeerId};
+
+use super::IsRequest;
+use crate::UnifiedReputationChange;
+
+mod error;
+pub use error::{Error, Fatal, NonFatal, Result};
+
+/// A request coming in, including a sender for sending responses.
+///
+/// Typed `IncomingRequest`s, see `IncomingRequest::get_config_receiver` and substrate
+/// `NetworkConfiguration` for more information.
+#[derive(Debug)]
+pub struct IncomingRequest<Req> {
+	/// `PeerId` of sending peer.
+	pub peer: PeerId,
+	/// The sent request.
+	pub payload: Req,
+	/// Sender for sending response back.
+	pub pending_response: OutgoingResponseSender<Req>,
+}
+
+impl<Req> IncomingRequest<Req>
+where
+	Req: IsRequest + Decode + Encode,
+	Req::Response: Encode,
+{
+	/// Create configuration for `NetworkConfiguration::request_response_porotocols` and a
+	/// corresponding typed receiver.
+	///
+	/// This Register that config with substrate networking and receive incoming requests via the
+	/// returned `IncomingRequestReceiver`.
+	pub fn get_config_receiver() -> (IncomingRequestReceiver<Req>, RequestResponseConfig) {
+		let (raw, cfg) = Req::PROTOCOL.get_config();
+		(IncomingRequestReceiver { raw, phantom: PhantomData {} }, cfg)
+	}
+
+	/// Create new `IncomingRequest`.
+	pub fn new(
+		peer: PeerId,
+		payload: Req,
+		pending_response: oneshot::Sender<netconfig::OutgoingResponse>,
+	) -> Self {
+		Self {
+			peer,
+			payload,
+			pending_response: OutgoingResponseSender { pending_response, phantom: PhantomData {} },
+		}
+	}
+
+	/// Try building from raw substrate request.
+	///
+	/// This function will fail if the request cannot be decoded and will apply passed in
+	/// reputation changes in that case.
+	///
+	/// Params:
+	///		- The raw request to decode
+	///		- Reputation changes to apply for the peer in case decoding fails.
+	fn try_from_raw(
+		raw: sc_network::config::IncomingRequest,
+		reputation_changes: Vec<UnifiedReputationChange>,
+	) -> std::result::Result<Self, NonFatal> {
+		let sc_network::config::IncomingRequest { payload, peer, pending_response } = raw;
+		let payload = match Req::decode(&mut payload.as_ref()) {
+			Ok(payload) => payload,
+			Err(err) => {
+				let reputation_changes =
+					reputation_changes.into_iter().map(|r| r.into_base_rep()).collect();
+				let response = sc_network::config::OutgoingResponse {
+					result: Err(()),
+					reputation_changes,
+					sent_feedback: None,
+				};
+
+				if let Err(_) = pending_response.send(response) {
+					return Err(NonFatal::DecodingErrorNoReputationChange(peer, err))
+				}
+				return Err(NonFatal::DecodingError(peer, err))
+			},
+		};
+		Ok(Self::new(peer, payload, pending_response))
+	}
+
+	/// Convert into raw untyped substrate `IncomingRequest`.
+	///
+	/// This is mostly useful for testing.
+	pub fn into_raw(self) -> sc_network::config::IncomingRequest {
+		sc_network::config::IncomingRequest {
+			peer: self.peer,
+			payload: self.payload.encode(),
+			pending_response: self.pending_response.pending_response,
+		}
+	}
+
+	/// Send the response back.
+	///
+	/// Calls [`OutgoingResponseSender::send_response`].
+	pub fn send_response(self, resp: Req::Response) -> std::result::Result<(), Req::Response> {
+		self.pending_response.send_response(resp)
+	}
+
+	/// Send response with additional options.
+	///
+	/// Calls [`OutgoingResponseSender::send_outgoing_response`].
+	pub fn send_outgoing_response(
+		self,
+		resp: OutgoingResponse<<Req as IsRequest>::Response>,
+	) -> std::result::Result<(), ()> {
+		self.pending_response.send_outgoing_response(resp)
+	}
+}
+
+/// Sender for sending back responses on an `IncomingRequest`.
+#[derive(Debug)]
+pub struct OutgoingResponseSender<Req> {
+	pending_response: oneshot::Sender<netconfig::OutgoingResponse>,
+	phantom: PhantomData<Req>,
+}
+
+impl<Req> OutgoingResponseSender<Req>
+where
+	Req: IsRequest + Decode,
+	Req::Response: Encode,
+{
+	/// Send the response back.
+	///
+	/// On success we return `Ok(())`, on error we return the not sent `Response`.
+	///
+	/// `netconfig::OutgoingResponse` exposes a way of modifying the peer's reputation. If needed we
+	/// can change this function to expose this feature as well.
+	pub fn send_response(self, resp: Req::Response) -> std::result::Result<(), Req::Response> {
+		self.pending_response
+			.send(netconfig::OutgoingResponse {
+				result: Ok(resp.encode()),
+				reputation_changes: Vec::new(),
+				sent_feedback: None,
+			})
+			.map_err(|_| resp)
+	}
+
+	/// Send response with additional options.
+	///
+	/// This variant allows for waiting for the response to be sent out, allows for changing peer's
+	/// reputation and allows for not sending a response at all (for only changing the peer's
+	/// reputation).
+	pub fn send_outgoing_response(
+		self,
+		resp: OutgoingResponse<<Req as IsRequest>::Response>,
+	) -> std::result::Result<(), ()> {
+		let OutgoingResponse { result, reputation_changes, sent_feedback } = resp;
+
+		let response = netconfig::OutgoingResponse {
+			result: result.map(|v| v.encode()),
+			reputation_changes: reputation_changes.into_iter().map(|c| c.into_base_rep()).collect(),
+			sent_feedback,
+		};
+
+		self.pending_response.send(response).map_err(|_| ())
+	}
+}
+
+/// Typed variant of [`netconfig::OutgoingResponse`].
+///
+/// Responses to `IncomingRequest`s.
+pub struct OutgoingResponse<Response> {
+	/// The payload of the response.
+	///
+	/// `Err(())` if none is available e.g. due to an error while handling the request.
+	pub result: std::result::Result<Response, ()>,
+
+	/// Reputation changes accrued while handling the request. To be applied to the reputation of
+	/// the peer sending the request.
+	pub reputation_changes: Vec<UnifiedReputationChange>,
+
+	/// If provided, the `oneshot::Sender` will be notified when the request has been sent to the
+	/// peer.
+	pub sent_feedback: Option<oneshot::Sender<()>>,
+}
+
+/// Receiver for incoming requests.
+///
+/// Takes care of decoding and handling of invalid encoded requests.
+pub struct IncomingRequestReceiver<Req> {
+	raw: mpsc::Receiver<netconfig::IncomingRequest>,
+	phantom: PhantomData<Req>,
+}
+
+impl<Req> IncomingRequestReceiver<Req>
+where
+	Req: IsRequest + Decode + Encode,
+	Req::Response: Encode,
+{
+	/// Try to receive the next incoming request.
+	///
+	/// Any received request will be decoded, on decoding errors the provided reputation changes
+	/// will be applied and an error will be reported.
+	pub async fn recv<F>(&mut self, reputation_changes: F) -> Result<IncomingRequest<Req>>
+	where
+		F: FnOnce() -> Vec<UnifiedReputationChange>,
+	{
+		let req = match self.raw.next().await {
+			None => return Err(Fatal::RequestChannelExhausted.into()),
+			Some(raw) => IncomingRequest::<Req>::try_from_raw(raw, reputation_changes())?,
+		};
+		Ok(req)
+	}
+}

polkadot/node/network/protocol/src/request_response/mod.rs

@@ -40,11 +40,14 @@ use strum::EnumIter;
 
 pub use sc_network::{config as network, config::RequestResponseConfig};
 
-/// All requests that can be sent to the network bridge.
-pub mod request;
-pub use request::{
-	IncomingRequest, OutgoingRequest, OutgoingResult, Recipient, Requests, ResponseSender,
-};
+/// Everything related to handling of incoming requests.
+pub mod incoming;
+/// Everything related to handling of outgoing requests.
+pub mod outgoing;
+
+pub use incoming::{IncomingRequest, IncomingRequestReceiver};
+
+pub use outgoing::{OutgoingRequest, OutgoingResult, Recipient, Requests, ResponseSender};
 
 ///// Multiplexer for incoming requests.
 // pub mod multiplexer;
@@ -248,3 +251,12 @@ impl Protocol {
 		}
 	}
 }
+
+/// Common properties of any `Request`.
+pub trait IsRequest {
+	/// Each request has a corresponding `Response`.
+	type Response;
+
+	/// What protocol this `Request` implements.
+	const PROTOCOL: Protocol;
+}

polkadot/node/network/protocol/src/request_response/outgoing.rs

@@ -14,32 +14,17 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::marker::PhantomData;
-
 use futures::{channel::oneshot, prelude::Future};
+use thiserror::Error;
 
 use parity_scale_codec::{Decode, Encode, Error as DecodingError};
+
 use sc_network as network;
-use sc_network::{config as netconfig, PeerId};
-use thiserror::Error;
+use sc_network::PeerId;
 
 use polkadot_primitives::v1::AuthorityDiscoveryId;
 
-use crate::UnifiedReputationChange;
-
-use super::{v1, Protocol};
-
-/// Used by the network to send us a response to a request.
-pub type ResponseSender = oneshot::Sender<Result<Vec<u8>, network::RequestFailure>>;
-
-/// Common properties of any `Request`.
-pub trait IsRequest {
-	/// Each request has a corresponding `Response`.
-	type Response;
-
-	/// What protocol this `Request` implements.
-	const PROTOCOL: Protocol;
-}
+use super::{v1, IsRequest, Protocol};
 
 /// All requests that can be sent to the network bridge via `NetworkBridgeMessage::SendRequest`.
 #[derive(Debug)]
@@ -90,13 +75,23 @@ impl Requests {
 	}
 }
 
-/// Potential recipients of an outgoing request.
-#[derive(Debug, Eq, Hash, PartialEq, Clone)]
-pub enum Recipient {
-	/// Recipient is a regular peer and we know its peer id.
-	Peer(PeerId),
-	/// Recipient is a validator, we address it via this `AuthorityDiscoveryId`.
-	Authority(AuthorityDiscoveryId),
+/// Used by the network to send us a response to a request.
+pub type ResponseSender = oneshot::Sender<Result<Vec<u8>, network::RequestFailure>>;
+
+/// Any error that can occur when sending a request.
+#[derive(Debug, Error)]
+pub enum RequestError {
+	/// Response could not be decoded.
+	#[error("Response could not be decoded")]
+	InvalidResponse(#[source] DecodingError),
+
+	/// Some error in substrate/libp2p happened.
+	#[error("Some network error occurred")]
+	NetworkError(#[source] network::RequestFailure),
+
+	/// Response got canceled by networking.
+	#[error("Response channel got canceled")]
+	Canceled(#[source] oneshot::Canceled),
 }
 
 /// A request to be sent to the network bridge, including a sender for sending responses/failures.
@@ -119,32 +114,13 @@ pub struct OutgoingRequest<Req> {
 	pub pending_response: ResponseSender,
 }
 
-/// Any error that can occur when sending a request.
-#[derive(Debug, Error)]
-pub enum RequestError {
-	/// Response could not be decoded.
-	#[error("Response could not be decoded")]
-	InvalidResponse(#[source] DecodingError),
-
-	/// Some error in substrate/libp2p happened.
-	#[error("Some network error occurred")]
-	NetworkError(#[source] network::RequestFailure),
-
-	/// Response got canceled by networking.
-	#[error("Response channel got canceled")]
-	Canceled(#[source] oneshot::Canceled),
-}
-
-/// Things that can go wrong when decoding an incoming request.
-#[derive(Debug, Error)]
-pub enum ReceiveError {
-	/// Decoding failed, we were able to change the peer's reputation accordingly.
-	#[error("Decoding request failed for peer {0}.")]
-	DecodingError(PeerId, #[source] DecodingError),
-
-	/// Decoding failed, but sending reputation change failed.
-	#[error("Decoding request failed for peer {0}, and changing reputation failed.")]
-	DecodingErrorNoReputationChange(PeerId, #[source] DecodingError),
+/// Potential recipients of an outgoing request.
+#[derive(Debug, Eq, Hash, PartialEq, Clone)]
+pub enum Recipient {
+	/// Recipient is a regular peer and we know its peer id.
+	Peer(PeerId),
+	/// Recipient is a validator, we address it via this `AuthorityDiscoveryId`.
+	Authority(AuthorityDiscoveryId),
 }
 
 /// Responses received for an `OutgoingRequest`.
@@ -179,6 +155,18 @@ where
 	}
 }
 
+/// Future for actually receiving a typed response for an `OutgoingRequest`.
+async fn receive_response<Req>(
+	rec: oneshot::Receiver<Result<Vec<u8>, network::RequestFailure>>,
+) -> OutgoingResult<Req::Response>
+where
+	Req: IsRequest,
+	Req::Response: Decode,
+{
+	let raw = rec.await??;
+	Ok(Decode::decode(&mut raw.as_ref())?)
+}
+
 impl From<DecodingError> for RequestError {
 	fn from(err: DecodingError) -> Self {
 		Self::InvalidResponse(err)
@@ -196,164 +184,3 @@ impl From<oneshot::Canceled> for RequestError {
 		Self::Canceled(err)
 	}
 }
-
-/// A request coming in, including a sender for sending responses.
-///
-/// `IncomingRequest`s are produced by `RequestMultiplexer` on behalf of the network bridge.
-#[derive(Debug)]
-pub struct IncomingRequest<Req> {
-	/// `PeerId` of sending peer.
-	pub peer: PeerId,
-	/// The sent request.
-	pub payload: Req,
-	/// Sender for sending response back.
-	pub pending_response: OutgoingResponseSender<Req>,
-}
-
-/// Sender for sending back responses on an `IncomingRequest`.
-#[derive(Debug)]
-pub struct OutgoingResponseSender<Req> {
-	pending_response: oneshot::Sender<netconfig::OutgoingResponse>,
-	phantom: PhantomData<Req>,
-}
-
-impl<Req> OutgoingResponseSender<Req>
-where
-	Req: IsRequest + Decode,
-	Req::Response: Encode,
-{
-	/// Send the response back.
-	///
-	/// On success we return `Ok(())`, on error we return the not sent `Response`.
-	///
-	/// `netconfig::OutgoingResponse` exposes a way of modifying the peer's reputation. If needed we
-	/// can change this function to expose this feature as well.
-	pub fn send_response(self, resp: Req::Response) -> Result<(), Req::Response> {
-		self.pending_response
-			.send(netconfig::OutgoingResponse {
-				result: Ok(resp.encode()),
-				reputation_changes: Vec::new(),
-				sent_feedback: None,
-			})
-			.map_err(|_| resp)
-	}
-
-	/// Send response with additional options.
-	///
-	/// This variant allows for waiting for the response to be sent out, allows for changing peer's
-	/// reputation and allows for not sending a response at all (for only changing the peer's
-	/// reputation).
-	pub fn send_outgoing_response(
-		self,
-		resp: OutgoingResponse<<Req as IsRequest>::Response>,
-	) -> Result<(), ()> {
-		let OutgoingResponse { result, reputation_changes, sent_feedback } = resp;
-
-		let response = netconfig::OutgoingResponse {
-			result: result.map(|v| v.encode()),
-			reputation_changes: reputation_changes.into_iter().map(|c| c.into_base_rep()).collect(),
-			sent_feedback,
-		};
-
-		self.pending_response.send(response).map_err(|_| ())
-	}
-}
-
-/// Typed variant of [`netconfig::OutgoingResponse`].
-///
-/// Responses to `IncomingRequest`s.
-pub struct OutgoingResponse<Response> {
-	/// The payload of the response.
-	///
-	/// `Err(())` if none is available e.g. due an error while handling the request.
-	pub result: Result<Response, ()>,
-
-	/// Reputation changes accrued while handling the request. To be applied to the reputation of
-	/// the peer sending the request.
-	pub reputation_changes: Vec<UnifiedReputationChange>,
-
-	/// If provided, the `oneshot::Sender` will be notified when the request has been sent to the
-	/// peer.
-	pub sent_feedback: Option<oneshot::Sender<()>>,
-}
-
-impl<Req> IncomingRequest<Req>
-where
-	Req: IsRequest + Decode,
-	Req::Response: Encode,
-{
-	/// Create new `IncomingRequest`.
-	pub fn new(
-		peer: PeerId,
-		payload: Req,
-		pending_response: oneshot::Sender<netconfig::OutgoingResponse>,
-	) -> Self {
-		Self {
-			peer,
-			payload,
-			pending_response: OutgoingResponseSender { pending_response, phantom: PhantomData {} },
-		}
-	}
-
-	/// Try building from raw substrate request.
-	///
-	/// This function will fail if the request cannot be decoded and will apply passed in
-	/// reputation changes in that case.
-	///
-	/// Params:
-	///		- The raw request to decode
-	///		- Reputation changes to apply for the peer in case decoding fails.
-	pub fn try_from_raw(
-		raw: sc_network::config::IncomingRequest,
-		reputation_changes: Vec<UnifiedReputationChange>,
-	) -> Result<Self, ReceiveError> {
-		let sc_network::config::IncomingRequest { payload, peer, pending_response } = raw;
-		let payload = match Req::decode(&mut payload.as_ref()) {
-			Ok(payload) => payload,
-			Err(err) => {
-				let reputation_changes =
-					reputation_changes.into_iter().map(|r| r.into_base_rep()).collect();
-				let response = sc_network::config::OutgoingResponse {
-					result: Err(()),
-					reputation_changes,
-					sent_feedback: None,
-				};
-
-				if let Err(_) = pending_response.send(response) {
-					return Err(ReceiveError::DecodingErrorNoReputationChange(peer, err))
-				}
-				return Err(ReceiveError::DecodingError(peer, err))
-			},
-		};
-		Ok(Self::new(peer, payload, pending_response))
-	}
-
-	/// Send the response back.
-	///
-	/// Calls [`OutgoingResponseSender::send_response`].
-	pub fn send_response(self, resp: Req::Response) -> Result<(), Req::Response> {
-		self.pending_response.send_response(resp)
-	}
-
-	/// Send response with additional options.
-	///
-	/// Calls [`OutgoingResponseSender::send_outgoing_response`].
-	pub fn send_outgoing_response(
-		self,
-		resp: OutgoingResponse<<Req as IsRequest>::Response>,
-	) -> Result<(), ()> {
-		self.pending_response.send_outgoing_response(resp)
-	}
-}
-
-/// Future for actually receiving a typed response for an `OutgoingRequest`.
-async fn receive_response<Req>(
-	rec: oneshot::Receiver<Result<Vec<u8>, network::RequestFailure>>,
-) -> OutgoingResult<Req::Response>
-where
-	Req: IsRequest,
-	Req::Response: Decode,
-{
-	let raw = rec.await??;
-	Ok(Decode::decode(&mut raw.as_ref())?)
-}

polkadot/node/network/protocol/src/request_response/v1.rs

@@ -25,7 +25,7 @@ use polkadot_primitives::v1::{
 	CandidateHash, CandidateReceipt, CommittedCandidateReceipt, Hash, Id as ParaId, ValidatorIndex,
 };
 
-use super::{request::IsRequest, Protocol};
+use super::{IsRequest, Protocol};
 
 /// Request an availability chunk.
 #[derive(Debug, Copy, Clone, Encode, Decode)]