Split Substrate messages into multiple substreams (#1796)

* Split Substrate messages into multiple substreams

* Add back Clogged event

substrate/core/network-libp2p/src/custom_proto/behaviour.rs

@@ -22,7 +22,7 @@ use crate::parse_str_addr;
 use fnv::{FnvHashMap, FnvHashSet};
 use futures::prelude::*;
 use libp2p::core::swarm::{ConnectedPoint, NetworkBehaviour, NetworkBehaviourAction, PollParameters};
-use libp2p::core::{protocols_handler::ProtocolsHandler, Multiaddr, PeerId};
+use libp2p::core::{protocols_handler::ProtocolsHandler, Endpoint, Multiaddr, PeerId};
 use log::{debug, trace, warn};
 use smallvec::SmallVec;
 use std::{cmp, error, io, marker::PhantomData, path::Path, time::Duration, time::Instant};
@@ -458,13 +458,13 @@ where
 			trace!(target: "sub-libp2p", "Enabling custom protocols with {:?} (active)", peer_id);
 			self.events.push(NetworkBehaviourAction::SendEvent {
 				peer_id: peer_id.clone(),
-				event: CustomProtosHandlerIn::EnableActive,
+				event: CustomProtosHandlerIn::Enable(Endpoint::Dialer),
 			});
 		} else {
 			trace!(target: "sub-libp2p", "Enabling custom protocols with {:?} (passive)", peer_id);
 			self.events.push(NetworkBehaviourAction::SendEvent {
 				peer_id: peer_id.clone(),
-				event: CustomProtosHandlerIn::EnablePassive,
+				event: CustomProtosHandlerIn::Enable(Endpoint::Listener),
 			});
 		}
 
@@ -581,10 +581,15 @@ where
 					messages,
 				}));
 			}
-			CustomProtosHandlerOut::ProtocolError { protocol_id, error } => {
-				warn!(target: "sub-libp2p", "Network misbehaviour from {:?} with protocol \
-					{:?}: {:?}", source, protocol_id, error);
-				self.ban_peer(source);
+			CustomProtosHandlerOut::ProtocolError { protocol_id, error, is_severe } => {
+				if is_severe {
+					warn!(target: "sub-libp2p", "Network misbehaviour from {:?} with protocol \
+						{:?}: {:?}", source, protocol_id, error);
+					self.ban_peer(source);
+				} else {
+					debug!(target: "sub-libp2p", "Network misbehaviour from {:?} with protocol \
+						{:?}: {:?}", source, protocol_id, error);
+				}
 			}
 		}
 	}

substrate/core/network-libp2p/src/custom_proto/mod.rs

@@ -15,7 +15,7 @@
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
 pub use self::behaviour::{CustomProtos, CustomProtosOut};
-pub use self::upgrade::{CustomMessage, RegisteredProtocol, RegisteredProtocols};
+pub use self::upgrade::{CustomMessage, CustomMessageId, RegisteredProtocol, RegisteredProtocols};
 
 mod behaviour;
 mod handler;

substrate/core/network-libp2p/src/custom_proto/topology.rs

@@ -415,7 +415,12 @@ impl NetTopology {
 					continue
 				}
 
-				debug_assert!(!a.is_connected());
+				// It is possible that we are connected to this address, and that the dial failure
+				// concerns another peer.
+				if a.is_connected() {
+					continue
+				}
+
 				a.adjust_score(SCORE_DIFF_ON_FAILED_TO_CONNECT);
 				trace!(target: "sub-libp2p", "Back off for {} = {:?}", addr, a.next_back_off);
 				a.back_off_until = Instant::now() + a.next_back_off;

substrate/core/network-libp2p/src/custom_proto/upgrade.rs

@@ -16,10 +16,10 @@
 
 use crate::ProtocolId;
 use bytes::Bytes;
-use libp2p::core::{UpgradeInfo, InboundUpgrade, OutboundUpgrade, upgrade::ProtocolName};
+use libp2p::core::{Endpoint, UpgradeInfo, InboundUpgrade, OutboundUpgrade, upgrade::ProtocolName};
 use libp2p::tokio_codec::Framed;
 use log::warn;
-use std::{collections::VecDeque, io, marker::PhantomData, vec::IntoIter as VecIntoIter};
+use std::{collections::VecDeque, io, iter, marker::PhantomData, vec::IntoIter as VecIntoIter};
 use futures::{prelude::*, future, stream};
 use tokio_io::{AsyncRead, AsyncWrite};
 use unsigned_varint::codec::UviBytes;
@@ -84,6 +84,9 @@ impl<TMessage> Clone for RegisteredProtocol<TMessage> {
 pub struct RegisteredProtocolSubstream<TMessage, TSubstream> {
 	/// If true, we are in the process of closing the sink.
 	is_closing: bool,
+	/// Whether the local node opened this substream (dialer), or we received this substream from
+	/// the remote (listener).
+	endpoint: Endpoint,
 	/// Buffer of packets to send.
 	send_queue: VecDeque<Vec<u8>>,
 	/// If true, we should call `poll_complete` on the inner sink.
@@ -97,6 +100,9 @@ pub struct RegisteredProtocolSubstream<TMessage, TSubstream> {
 	/// If true, we have sent a "remote is clogged" event recently and shouldn't send another one
 	/// unless the buffer empties then fills itself again.
 	clogged_fuse: bool,
+	/// If true, then this substream uses the "/multi/" version of the protocol. This is a hint
+	/// that the handler can behave differently.
+	is_multiplex: bool,
 	/// Marker to pin the generic.
 	marker: PhantomData<TMessage>,
 }
@@ -114,6 +120,18 @@ impl<TMessage, TSubstream> RegisteredProtocolSubstream<TMessage, TSubstream> {
 		self.protocol_version
 	}
 
+	/// Returns whether the local node opened this substream (dialer), or we received this
+	/// substream from the remote (listener).
+	pub fn endpoint(&self) -> Endpoint {
+		self.endpoint
+	}
+
+	/// Returns true if we negotiated the "multiplexed" version. This means that the handler can
+	/// open multiple substreams instead of just one.
+	pub fn is_multiplex(&self) -> bool {
+		self.is_multiplex
+	}
+
 	/// Starts a graceful shutdown process on this substream.
 	///
 	/// Note that "graceful" means that we sent a closing message. We don't wait for any
@@ -138,14 +156,39 @@ impl<TMessage, TSubstream> RegisteredProtocolSubstream<TMessage, TSubstream> {
 
 /// Implemented on messages that can be sent or received on the network.
 pub trait CustomMessage {
-	/// Turns a message into raw bytes.
+	/// Turns a message into the raw bytes to send over the network.
 	fn into_bytes(self) -> Vec<u8>;
-	/// Tries to part `bytes` into a message.
+
+	/// Tries to parse `bytes` received from the network into a message.
 	fn from_bytes(bytes: &[u8]) -> Result<Self, ()>
 		where Self: Sized;
+
+	/// Returns a unique ID that is used to match request and responses.
+	///
+	/// The networking layer employs multiplexing in order to have multiple parallel data streams.
+	/// Transmitting messages over the network uses two kinds of substreams:
+	///
+	/// - Undirectional substreams, where we send a single message then close the substream.
+	/// - Bidirectional substreams, where we send a message then wait for a response. Once the
+	///   response has arrived, we close the substream.
+	///
+	/// If `request_id()` returns `OneWay`, then this message will be sent or received over a
+	/// unidirectional substream. If instead it returns `Request` or `Response`, then we use the
+	/// value to match a request with its response.
+	fn request_id(&self) -> CustomMessageId;
 }
 
-/// This trait implementation exists mostly for testing convenience.
+/// See the documentation of `CustomMessage::request_id`.
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum CustomMessageId {
+	OneWay,
+	Request(u64),
+	Response(u64),
+}
+
+// These trait implementations exist mostly for testing convenience. This should eventually be
+// removed.
+
 impl CustomMessage for Vec<u8> {
 	fn into_bytes(self) -> Vec<u8> {
 		self
@@ -154,6 +197,45 @@ impl CustomMessage for Vec<u8> {
 	fn from_bytes(bytes: &[u8]) -> Result<Self, ()> {
 		Ok(bytes.to_vec())
 	}
+
+	fn request_id(&self) -> CustomMessageId {
+		CustomMessageId::OneWay
+	}
+}
+
+impl CustomMessage for (Option<u64>, Vec<u8>) {
+	fn into_bytes(self) -> Vec<u8> {
+		use byteorder::WriteBytesExt;
+		use std::io::Write;
+		let mut out = Vec::new();
+		out.write_u64::<byteorder::BigEndian>(self.0.unwrap_or(u64::max_value()))
+			.expect("Writing to a Vec can never fail");
+		out.write_all(&self.1).expect("Writing to a Vec can never fail");
+		out
+	}
+
+	fn from_bytes(bytes: &[u8]) -> Result<Self, ()> {
+		use byteorder::ReadBytesExt;
+		use std::io::Read;
+		let mut rdr = std::io::Cursor::new(bytes);
+		let id = rdr.read_u64::<byteorder::BigEndian>().map_err(|_| ())?;
+		let mut out = Vec::new();
+		rdr.read_to_end(&mut out).map_err(|_| ())?;
+		let id = if id == u64::max_value() {
+			None
+		} else {
+			Some(id)
+		};
+		Ok((id, out))
+	}
+
+	fn request_id(&self) -> CustomMessageId {
+		if let Some(id) = self.0 {
+			CustomMessageId::Request(id)
+		} else {
+			CustomMessageId::OneWay
+		}
+	}
 }
 
 /// Event produced by the `RegisteredProtocolSubstream`.
@@ -176,11 +258,6 @@ where TSubstream: AsyncRead + AsyncWrite, TMessage: CustomMessage {
 	type Error = io::Error;
 
 	fn poll(&mut self) -> Poll<Option<Self::Item>, Self::Error> {
-		// If we are closing, close as soon as the Sink is closed.
-		if self.is_closing {
-			return Ok(self.inner.close()?.map(|()| None))
-		}
-
 		// Flushing the local queue.
 		while let Some(packet) = self.send_queue.pop_front() {
 			match self.inner.start_send(packet)? {
@@ -192,6 +269,11 @@ where TSubstream: AsyncRead + AsyncWrite, TMessage: CustomMessage {
 			}
 		}
 
+		// If we are closing, close as soon as the Sink is closed.
+		if self.is_closing {
+			return Ok(self.inner.close()?.map(|()| None))
+		}
+
 		// Indicating that the remote is clogged if that's the case.
 		if self.send_queue.len() >= 2048 {
 			if !self.clogged_fuse {
@@ -227,13 +309,13 @@ where TSubstream: AsyncRead + AsyncWrite, TMessage: CustomMessage {
 						io::ErrorKind::InvalidData
 					})?;
 				Ok(Async::Ready(Some(RegisteredProtocolEvent::Message(message))))
-			},
+			}
 			Async::Ready(None) =>
 				if !self.requires_poll_complete && self.send_queue.is_empty() {
 					Ok(Async::Ready(None))
 				} else {
 					Ok(Async::NotReady)
-				},
+				}
 			Async::NotReady => Ok(Async::NotReady),
 		}
 	}
@@ -246,14 +328,33 @@ impl<TMessage> UpgradeInfo for RegisteredProtocol<TMessage> {
 	#[inline]
 	fn protocol_info(&self) -> Self::InfoIter {
 		// Report each version as an individual protocol.
-		self.supported_versions.iter().map(|&version| {
+		self.supported_versions.iter().flat_map(|&version| {
 			let num = version.to_string();
-			let mut name = self.base_name.clone();
-			name.extend_from_slice(num.as_bytes());
-			RegisteredProtocolName {
-				name,
+
+			// Note that `name1` is the multiplex version, as we priviledge it over the old one.
+			let mut name1 = self.base_name.clone();
+			name1.extend_from_slice(b"multi/");
+			name1.extend_from_slice(num.as_bytes());
+			let proto1 = RegisteredProtocolName {
+				name: name1,
 				version,
-			}
+				is_multiplex: true,
+			};
+
+			let mut name2 = self.base_name.clone();
+			name2.extend_from_slice(num.as_bytes());
+			let proto2 = RegisteredProtocolName {
+				name: name2,
+				version,
+				is_multiplex: false,
+			};
+
+			// Important note: we prioritize the backwards compatible mode for now.
+			// After some intensive testing has been done, we should switch to the new mode by
+			// default.
+			// Then finally we can remove the old mode after everyone has switched.
+			// See https://github.com/paritytech/substrate/issues/1692
+			iter::once(proto2).chain(iter::once(proto1))
 		}).collect::<Vec<_>>().into_iter()
 	}
 }
@@ -265,6 +366,8 @@ pub struct RegisteredProtocolName {
 	name: Bytes,
 	/// Version number. Stored in string form in `name`, but duplicated here for easier retrieval.
 	version: u8,
+	/// If true, then this version is the one with the multiplexing.
+	is_multiplex: bool,
 }
 
 impl ProtocolName for RegisteredProtocolName {
@@ -289,12 +392,14 @@ where TSubstream: AsyncRead + AsyncWrite,
 
 		future::ok(RegisteredProtocolSubstream {
 			is_closing: false,
+			endpoint: Endpoint::Listener,
 			send_queue: VecDeque::new(),
 			requires_poll_complete: false,
 			inner: framed.fuse(),
 			protocol_id: self.id,
 			protocol_version: info.version,
 			clogged_fuse: false,
+			is_multiplex: info.is_multiplex,
 			marker: PhantomData,
 		})
 	}
@@ -312,8 +417,20 @@ where TSubstream: AsyncRead + AsyncWrite,
 		socket: TSubstream,
 		info: Self::Info,
 	) -> Self::Future {
-		// Upgrades are symmetrical.
-		self.upgrade_inbound(socket, info)
+		let framed = Framed::new(socket, UviBytes::default());
+
+		future::ok(RegisteredProtocolSubstream {
+			is_closing: false,
+			endpoint: Endpoint::Dialer,
+			send_queue: VecDeque::new(),
+			requires_poll_complete: false,
+			inner: framed.fuse(),
+			protocol_id: self.id,
+			protocol_version: info.version,
+			clogged_fuse: false,
+			is_multiplex: info.is_multiplex,
+			marker: PhantomData,
+		})
 	}
 }
 
@@ -326,11 +443,6 @@ impl<TMessage> RegisteredProtocols<TMessage> {
 	pub fn len(&self) -> usize {
 		self.0.len()
 	}
-
-	/// Returns true if the given protocol is in the list.
-	pub fn has_protocol(&self, protocol: ProtocolId) -> bool {
-		self.0.iter().any(|p| p.id == protocol)
-	}
 }
 
 impl<TMessage> Default for RegisteredProtocols<TMessage> {

substrate/core/network-libp2p/src/lib.rs

@@ -24,7 +24,7 @@ mod service_task;
 mod traits;
 mod transport;
 
-pub use crate::custom_proto::{CustomMessage, RegisteredProtocol};
+pub use crate::custom_proto::{CustomMessage, CustomMessageId, RegisteredProtocol};
 pub use crate::error::{Error, ErrorKind, DisconnectReason};
 pub use crate::secret::obtain_private_key;
 pub use crate::service_task::{start_service, Service, ServiceEvent};