Generic request/response infrastructure for Polkadot (#2352)

* Move NetworkBridgeEvent to subsystem::messages.

It is not protocol related at all, it is in fact only part of the
subsystem communication as it gets wrapped into messages of each
subsystem.

* Request/response infrastructure is taking shape.

WIP: Does not compile.

* Multiplexer variant not supported by Rusts type system.

* request_response::request type checks.

* Cleanup.

* Minor fixes for request_response.

* Implement request sending + move multiplexer.

Request multiplexer is moved to bridge, as there the implementation is
more straight forward as we can specialize on `AllMessages` for the
multiplexing target.

Sending of requests is mostly complete, apart from a few `From`
instances. Receiving is also almost done, initializtion needs to be
fixed and the multiplexer needs to be invoked.

* Remove obsolete multiplexer.

* Initialize bridge with multiplexer.

* Finish generic request sending/receiving.

Subsystems are now able to receive and send requests and responses via
the overseer.

* Doc update.

* Fixes.

* Link issue for not yet implemented code.

* Fixes suggested by @ordian - thanks!

- start encoding at 0
- don't crash on zero protocols
- don't panic on not yet implemented request handling

* Update node/network/protocol/src/request_response/v1.rs

Use index 0 instead of 1.

Co-authored-by: Andronik Ordian <write@reusable.software>

* Update node/network/protocol/src/request_response.rs

Co-authored-by: Andronik Ordian <write@reusable.software>

* Fix existing tests.

* Better avoidance of division by zoro errors.

* Doc fixes.

* send_request -> start_request.

* Fix missing renamings.

* Update substrate.

* Pass TryConnect instead of true.

* Actually import `IfDisconnected`.

* Fix wrong import.

* Update node/network/bridge/src/lib.rs

typo

Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>

* Update node/network/bridge/src/multiplexer.rs

Remove redundant import.

Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>

* Stop doing tracing from within `From` instance.

Thanks for the catch @tomaka!

* Get rid of redundant import.

* Formatting cleanup.

* Fix tests.

* Add link to issue.

* Clarify comments some more.

* Fix tests.

* Formatting fix.

* tabs

* Fix link

Co-authored-by: Bernhard Schuster <bernhard@ahoi.io>

* Use map_err.

Co-authored-by: Bernhard Schuster <bernhard@ahoi.io>

* Improvements inspired by suggestions by @drahnr.

- Channel size is now determined by function.
- Explicitely scope NetworkService::start_request.

Co-authored-by: Andronik Ordian <write@reusable.software>
Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>
Co-authored-by: Bernhard Schuster <bernhard@ahoi.io>

polkadot/node/network/protocol/src/lib.rs

@@ -21,7 +21,7 @@
 
 use polkadot_primitives::v1::{Hash, BlockNumber};
 use parity_scale_codec::{Encode, Decode};
-use std::{convert::TryFrom, fmt, collections::HashMap};
+use std::{fmt, collections::HashMap};
 
 pub use sc_network::{ReputationChange, PeerId};
 #[doc(hidden)]
@@ -33,6 +33,9 @@ pub use std::sync::Arc;
 /// Peer-sets and protocols used for parachains.
 pub mod peer_set;
 
+/// Request/response protocols used in Polkadot.
+pub mod request_response;
+
 /// A unique identifier of a request.
 pub type RequestId = u64;
 
@@ -85,25 +88,6 @@ impl Into<sc_network::ObservedRole> for ObservedRole {
 	}
 }
 
-/// Events from network.
-#[derive(Debug, Clone, PartialEq)]
-pub enum NetworkBridgeEvent<M> {
-	/// A peer has connected.
-	PeerConnected(PeerId, ObservedRole),
-
-	/// A peer has disconnected.
-	PeerDisconnected(PeerId),
-
-	/// Peer has sent a message.
-	PeerMessage(PeerId, M),
-
-	/// Peer's `View` has changed.
-	PeerViewChange(PeerId, View),
-
-	/// Our view has changed.
-	OurViewChange(OurView),
-}
-
 macro_rules! impl_try_from {
 	($m_ty:ident, $variant:ident, $out:ty) => {
 		impl TryFrom<$m_ty> for $out {
@@ -132,29 +116,6 @@ macro_rules! impl_try_from {
 	}
 }
 
-impl<M> NetworkBridgeEvent<M> {
-	/// Focus an overarching network-bridge event into some more specific variant.
-	///
-	/// This acts as a call to `clone`, except in the case where the event is a message event,
-	/// in which case the clone can be expensive and it only clones if the message type can
-	/// be focused.
-	pub fn focus<'a, T>(&'a self) -> Result<NetworkBridgeEvent<T>, WrongVariant>
-		where T: 'a + Clone, &'a T: TryFrom<&'a M, Error = WrongVariant>
-	{
-		Ok(match *self {
-			NetworkBridgeEvent::PeerConnected(ref peer, ref role)
-				=> NetworkBridgeEvent::PeerConnected(peer.clone(), role.clone()),
-			NetworkBridgeEvent::PeerDisconnected(ref peer)
-				=> NetworkBridgeEvent::PeerDisconnected(peer.clone()),
-			NetworkBridgeEvent::PeerMessage(ref peer, ref msg)
-				=> NetworkBridgeEvent::PeerMessage(peer.clone(), <&'a T>::try_from(msg)?.clone()),
-			NetworkBridgeEvent::PeerViewChange(ref peer, ref view)
-				=> NetworkBridgeEvent::PeerViewChange(peer.clone(), view.clone()),
-			NetworkBridgeEvent::OurViewChange(ref view)
-				=> NetworkBridgeEvent::OurViewChange(view.clone()),
-		})
-	}
-}
 
 /// Specialized wrapper around [`View`].
 ///

polkadot/node/network/protocol/src/peer_set.rs

@@ -23,7 +23,8 @@ use strum::{EnumIter, IntoEnumIterator};
 /// The peer-sets and thus the protocols which are used for the network.
 #[derive(Debug, Clone, Copy, PartialEq, EnumIter)]
 pub enum PeerSet {
-	/// The validation peer-set is responsible for all messages related to candidate validation and communication among validators.
+	/// The validation peer-set is responsible for all messages related to candidate validation and
+	/// communication among validators.
 	Validation,
 	/// The collation peer-set is used for validator<>collator communication.
 	Collation,

polkadot/node/network/protocol/src/request_response.rs

@@ -0,0 +1,119 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Overview over request/responses as used in `Polkadot`.
+//!
+//! enum  Protocol .... List of all supported protocols.
+//!
+//! enum  Requests  .... List of all supported requests, each entry matches one in protocols, but
+//! has the actual request as payload.
+//!
+//! struct IncomingRequest .... wrapper for incoming requests, containing a sender for sending
+//! responses.
+//!
+//! struct OutgoingRequest .... wrapper for outgoing requests, containing a sender used by the
+//! networking code for delivering responses/delivery errors.
+//!
+//! trait `IsRequest` .... A trait describing a particular request. It is used for gathering meta
+//! data, like what is the corresponding response type.
+//!
+//!  Versioned (v1 module): The actual requests and responses as sent over the network.
+
+use std::borrow::Cow;
+use std::time::Duration;
+
+use futures::channel::mpsc;
+use strum::EnumIter;
+
+pub use sc_network::config as network;
+pub use sc_network::config::RequestResponseConfig;
+
+/// All requests that can be sent to the network bridge.
+pub mod request;
+pub use request::{IncomingRequest, OutgoingRequest, Requests};
+
+///// Multiplexer for incoming requests.
+// pub mod multiplexer;
+
+/// Actual versioned requests and responses, that are sent over the wire.
+pub mod v1;
+
+/// A protocol per subsystem seems to make the most sense, this way we don't need any dispatching
+/// within protocols.
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, EnumIter)]
+pub enum Protocol {
+	/// Protocol for availability fetching, used by availability distribution.
+	AvailabilityFetching,
+}
+
+/// Default request timeout in seconds.
+const DEFAULT_REQUEST_TIMEOUT: u64 = 8; 
+
+impl Protocol {
+	/// Get a configuration for a given Request response protocol.
+	///
+	/// Returns a receiver for messages received on this protocol and the requested
+	/// `ProtocolConfig`.
+	///
+	/// See also `dispatcher::RequestDispatcher`,  which makes use of this function and provides a more
+	/// high-level interface.
+	pub fn get_config(
+		self,
+	) -> (
+		mpsc::Receiver<network::IncomingRequest>,
+		RequestResponseConfig,
+	) {
+		let p_name = self.into_protocol_name();
+		let (tx, rx) = mpsc::channel(self.get_channel_size());
+		let cfg = match self {
+			Protocol::AvailabilityFetching => RequestResponseConfig {
+				name: p_name,
+				// Arbitrary very conservative numbers:
+				// TODO: Get better numbers, see https://github.com/paritytech/polkadot/issues/2370
+				max_request_size: 10_000,
+				max_response_size: 1_000_000,
+				// Also just some relative conservative guess:
+				request_timeout: Duration::from_secs(DEFAULT_REQUEST_TIMEOUT),
+				inbound_queue: Some(tx),
+			},
+		};
+		(rx, cfg)
+	}
+
+	// Channel sizes for the supported protocols.
+	fn get_channel_size(self) -> usize {
+		match self {
+			// Hundreds of validators will start requesting their chunks once they see a candidate
+			// awaiting availability on chain. Given that they will see that block at different
+			// times (due to network delays), 100 seems big enough to accomodate for "bursts",
+			// assuming we can service requests relatively quickly, which would need to be measured
+			// as well.
+			Protocol::AvailabilityFetching => 100,
+		}
+	}
+
+	/// Get the protocol name of this protocol, as understood by substrate networking.
+	pub fn into_protocol_name(self) -> Cow<'static, str> {
+		self.get_protocol_name_static().into()
+	}
+
+	/// Get the protocol name associated with each peer set as static str.
+	pub const fn get_protocol_name_static(self) -> &'static str {
+		match self {
+			Protocol::AvailabilityFetching => "/polkadot/req_availability/1",
+		}
+	}
+}

polkadot/node/network/protocol/src/request_response/request.rs

@@ -0,0 +1,209 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use futures::channel::oneshot;
+use futures::prelude::Future;
+
+use parity_scale_codec::{Decode, Encode, Error as DecodingError};
+use sc_network as network;
+use sc_network::config as netconfig;
+use sc_network::PeerId;
+
+use super::{v1, Protocol};
+
+/// Common properties of any `Request`.
+pub trait IsRequest {
+	/// Each request has a corresponding `Response`.
+	type Response;
+
+	/// What protocol this `Request` implements.
+	const PROTOCOL: Protocol;
+}
+
+/// All requests that can be sent to the network bridge via `NetworkBridgeMessage::SendRequest`.
+#[derive(Debug)]
+pub enum Requests {
+	/// Request an availability chunk from a node.
+	AvailabilityFetching(OutgoingRequest<v1::AvailabilityFetchingRequest>),
+}
+
+impl Requests {
+	/// Get the protocol this request conforms to.
+	pub fn get_protocol(&self) -> Protocol {
+		match self {
+			Self::AvailabilityFetching(_) => Protocol::AvailabilityFetching,
+		}
+	}
+
+	/// Encode the request.
+	///
+	/// The corresponding protocol is returned as well, as we are now leaving typed territory.
+	///
+	/// Note: `Requests` is just an enum collecting all supported requests supported by network
+	/// bridge, it is never sent over the wire. This function just encodes the individual requests
+	/// contained in the enum.
+	pub fn encode_request(self) -> (Protocol, OutgoingRequest<Vec<u8>>) {
+		match self {
+			Self::AvailabilityFetching(r) => r.encode_request(),
+		}
+	}
+}
+
+/// A request to be sent to the network bridge, including a sender for sending responses/failures.
+///
+/// The network implementation will make use of that sender for informing the requesting subsystem
+/// about responses/errors.
+#[derive(Debug)]
+pub struct OutgoingRequest<Req> {
+	/// Intendent recipient of this request.
+	pub peer: PeerId,
+	/// The actual request to send over the wire.
+	pub payload: Req,
+	/// Sender which is used by networking to get us back a response.
+	pub pending_response: oneshot::Sender<Result<Vec<u8>, network::RequestFailure>>,
+}
+
+/// Any error that can occur when sending a request.
+pub enum RequestError {
+	/// Response could not be decoded.
+	InvalidResponse(DecodingError),
+
+	/// Some error in substrate/libp2p happened.
+	NetworkError(network::RequestFailure),
+
+	/// Response got canceled by networking.
+	Canceled(oneshot::Canceled),
+}
+
+impl<Req> OutgoingRequest<Req>
+where
+	Req: IsRequest + Encode,
+	Req::Response: Decode,
+{
+	/// Create a new `OutgoingRequest`.
+	///
+	/// It will contain a sender that is used by the networking for sending back responses. The
+	/// connected receiver is returned as the second element in the returned tuple.
+	pub fn new(
+		peer: PeerId,
+		payload: Req,
+	) -> (
+		Self,
+		impl Future<Output = Result<Req::Response, RequestError>>,
+	) {
+		let (tx, rx) = oneshot::channel();
+		let r = Self {
+			peer,
+			payload,
+			pending_response: tx,
+		};
+		(r, receive_response::<Req>(rx))
+	}
+
+	/// Encode a request into a `Vec<u8>`.
+	///
+	/// As this throws away type information, we also return the `Protocol` this encoded request
+	/// adheres to.
+	pub fn encode_request(self) -> (Protocol, OutgoingRequest<Vec<u8>>) {
+		let OutgoingRequest {
+			peer,
+			payload,
+			pending_response,
+		} = self;
+		let encoded = OutgoingRequest {
+			peer,
+			payload: payload.encode(),
+			pending_response,
+		};
+		(Req::PROTOCOL, encoded)
+	}
+}
+
+impl From<DecodingError> for RequestError {
+	fn from(err: DecodingError) -> Self {
+		Self::InvalidResponse(err)
+	}
+}
+
+impl From<network::RequestFailure> for RequestError {
+	fn from(err: network::RequestFailure) -> Self {
+		Self::NetworkError(err)
+	}
+}
+
+impl From<oneshot::Canceled> for RequestError {
+	fn from(err: oneshot::Canceled) -> Self {
+		Self::Canceled(err)
+	}
+}
+
+/// A request coming in, including a sender for sending responses.
+///
+/// `IncomingRequest`s are produced by `RequestMultiplexer` on behalf of the network bridge.
+#[derive(Debug)]
+pub struct IncomingRequest<Req> {
+	/// PeerId of sending peer.
+	pub peer: PeerId,
+	/// The sent request.
+	pub payload: Req,
+	pending_response: oneshot::Sender<netconfig::OutgoingResponse>,
+}
+
+impl<Req> IncomingRequest<Req>
+where
+	Req: IsRequest,
+	Req::Response: Encode,
+{
+	/// Create new `IncomingRequest`.
+	pub fn new(
+		peer: PeerId,
+		payload: Req,
+		pending_response: oneshot::Sender<netconfig::OutgoingResponse>,
+	) -> Self {
+		Self {
+			peer,
+			payload,
+			pending_response,
+		}
+	}
+
+	/// Send the response back.
+	///
+	/// On success we return Ok(()), on error we return the not sent `Response`.
+	///
+	/// netconfig::OutgoingResponse exposes a way of modifying the peer's reputation. If needed we
+	/// can change this function to expose this feature as well.
+	pub fn send_response(self, resp: Req::Response) -> Result<(), Req::Response> {
+		self.pending_response
+			.send(netconfig::OutgoingResponse {
+				result: Ok(resp.encode()),
+				reputation_changes: Vec::new(),
+			})
+			.map_err(|_| resp)
+	}
+}
+
+/// Future for actually receiving a typed response for an OutgoingRequest.
+async fn receive_response<Req>(
+	rec: oneshot::Receiver<Result<Vec<u8>, network::RequestFailure>>,
+) -> Result<Req::Response, RequestError>
+where
+	Req: IsRequest,
+	Req::Response: Decode,
+{
+	let raw = rec.await??;
+	Ok(Decode::decode(&mut raw.as_ref())?)
+}

polkadot/node/network/protocol/src/request_response/v1.rs

@@ -0,0 +1,44 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Requests and responses as sent over the wire for the individual protocols.
+
+use parity_scale_codec::{Decode, Encode};
+
+use polkadot_primitives::v1::{CandidateHash, ErasureChunk, ValidatorIndex};
+
+use super::request::IsRequest;
+use super::Protocol;
+
+/// Request an availability chunk.
+#[derive(Debug, Clone, Encode, Decode)]
+pub struct AvailabilityFetchingRequest {
+	candidate_hash: CandidateHash,
+	index: ValidatorIndex,
+}
+
+/// Receive a rqeuested erasure chunk.
+#[derive(Debug, Clone, Encode, Decode)]
+pub enum AvailabilityFetchingResponse {
+	/// The requested chunk.
+	#[codec(index = 0)]
+	Chunk(ErasureChunk),
+}
+
+impl IsRequest for AvailabilityFetchingRequest {
+	type Response = AvailabilityFetchingResponse;
+	const PROTOCOL: Protocol = Protocol::AvailabilityFetching;
+}