Open one substream for each notifications protocol (#4909)

* Open one substream for each notifications protocol

* Fix WASM build

* Apply suggestions from code review

Co-Authored-By: Toralf Wittner <tw@dtex.org>

* Address concerns

* Use unsigned-varint to read the varint

* Use unsigned-varint

* Forgot Cargo.lock

Co-authored-by: Toralf Wittner <tw@dtex.org>

substrate/client/network/src/protocol/generic_proto/handler/group.rs

@@ -0,0 +1,523 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Implementations of the `IntoProtocolsHandler` and `ProtocolsHandler` traits for both incoming
+//! and outgoing substreams for all gossiping protocols together.
+//!
+//! This is the main implementation of `ProtocolsHandler` in this crate, that handles all the
+//! protocols that are Substrate-related and outside of the scope of libp2p.
+//!
+//! # Usage
+//!
+//! The handler can be in one of the following states: `Initial`, `Enabled`, `Disabled`.
+//!
+//! The `Initial` state is the state that the handler initially is in. It is a temporary state
+//! during which the user must either enable or disable the handler. After that, the handler stays
+//! either enabled or disabled.
+//!
+//! On the wire, we try to open the following substreams:
+//!
+//! - One substream for each notification protocol passed as parameter to the
+//!   `NotifsHandlerProto::new` function.
+//! - One "legacy" substream used for anything non-related to gossiping, and used as a fallback
+//!   in case the notification protocol can't be opened.
+//!
+//! When the handler is in the `Enabled` state, we immediately open and try to maintain all the
+//! aforementioned substreams. When the handler is in the `Disabled` state, we immediately close
+//! (or abort opening) all these substreams. It is intended that in the future we allow states in
+//! which some protocols are open and not others. Symmetrically, we allow incoming
+//! Substrate-related substreams if and only if we are in the `Enabled` state.
+//!
+//! The user has the choice between sending a message with `SendNotification`, to send a
+//! notification, and `SendLegacy`, to send any other kind of message.
+//!
+
+use crate::protocol::generic_proto::{
+	handler::legacy::{LegacyProtoHandler, LegacyProtoHandlerProto, LegacyProtoHandlerIn, LegacyProtoHandlerOut},
+	handler::notif_in::{NotifsInHandlerProto, NotifsInHandler, NotifsInHandlerIn, NotifsInHandlerOut},
+	handler::notif_out::{NotifsOutHandlerProto, NotifsOutHandler, NotifsOutHandlerIn, NotifsOutHandlerOut},
+	upgrade::{NotificationsIn, NotificationsOut, NotificationsHandshakeError, RegisteredProtocol, UpgradeCollec},
+};
+use crate::protocol::message::generic::{Message as GenericMessage, ConsensusMessage};
+
+use bytes::BytesMut;
+use codec::Encode as _;
+use libp2p::core::{either::{EitherError, EitherOutput}, ConnectedPoint, PeerId};
+use libp2p::core::upgrade::{EitherUpgrade, UpgradeError, SelectUpgrade, InboundUpgrade, OutboundUpgrade};
+use libp2p::swarm::{
+	ProtocolsHandler, ProtocolsHandlerEvent,
+	IntoProtocolsHandler,
+	KeepAlive,
+	ProtocolsHandlerUpgrErr,
+	SubstreamProtocol,
+	NegotiatedSubstream,
+};
+use log::error;
+use sp_runtime::ConsensusEngineId;
+use std::{borrow::Cow, error, io, task::{Context, Poll}};
+
+/// Implements the `IntoProtocolsHandler` trait of libp2p.
+///
+/// Every time a connection with a remote starts, an instance of this struct is created and
+/// sent to a background task dedicated to this connection. Once the connection is established,
+/// it is turned into a [`NotifsHandler`].
+///
+/// See the documentation at the module level for more information.
+pub struct NotifsHandlerProto {
+	/// Prototypes for handlers for inbound substreams.
+	in_handlers: Vec<(NotifsInHandlerProto, ConsensusEngineId)>,
+
+	/// Prototypes for handlers for outbound substreams.
+	out_handlers: Vec<(NotifsOutHandlerProto, ConsensusEngineId)>,
+
+	/// Prototype for handler for backwards-compatibility.
+	legacy: LegacyProtoHandlerProto,
+}
+
+/// The actual handler once the connection has been established.
+///
+/// See the documentation at the module level for more information.
+pub struct NotifsHandler {
+	/// Handlers for inbound substreams.
+	in_handlers: Vec<(NotifsInHandler, ConsensusEngineId)>,
+
+	/// Handlers for outbound substreams.
+	out_handlers: Vec<(NotifsOutHandler, ConsensusEngineId)>,
+
+	/// Handler for backwards-compatibility.
+	legacy: LegacyProtoHandler,
+
+	/// State of this handler.
+	enabled: EnabledState,
+
+	/// If we receive inbound substream requests while in initialization mode,
+	/// we push the corresponding index here and process them when the handler
+	/// gets enabled/disabled.
+	pending_in: Vec<usize>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+enum EnabledState {
+	Initial,
+	Enabled,
+	Disabled,
+}
+
+impl IntoProtocolsHandler for NotifsHandlerProto {
+	type Handler = NotifsHandler;
+
+	fn inbound_protocol(&self) -> SelectUpgrade<UpgradeCollec<NotificationsIn>, RegisteredProtocol> {
+		let in_handlers = self.in_handlers.iter()
+			.map(|(h, _)| h.inbound_protocol())
+			.collect::<UpgradeCollec<_>>();
+
+		SelectUpgrade::new(in_handlers, self.legacy.inbound_protocol())
+	}
+
+	fn into_handler(self, remote_peer_id: &PeerId, connected_point: &ConnectedPoint) -> Self::Handler {
+		NotifsHandler {
+			in_handlers: self.in_handlers
+				.into_iter()
+				.map(|(p, e)| (p.into_handler(remote_peer_id, connected_point), e))
+				.collect(),
+			out_handlers: self.out_handlers
+				.into_iter()
+				.map(|(p, e)| (p.into_handler(remote_peer_id, connected_point), e))
+				.collect(),
+			legacy: self.legacy.into_handler(remote_peer_id, connected_point),
+			enabled: EnabledState::Initial,
+			pending_in: Vec::new(),
+		}
+	}
+}
+
+/// Event that can be received by a `NotifsHandler`.
+#[derive(Debug)]
+pub enum NotifsHandlerIn {
+	/// The node should start using custom protocols.
+	Enable,
+
+	/// The node should stop using custom protocols.
+	Disable,
+
+	/// Sends a message through the custom protocol substream.
+	///
+	/// > **Note**: This must **not** be an encoded `ConsensusMessage` message.
+	SendLegacy {
+		/// The message to send.
+		message: Vec<u8>,
+	},
+
+	/// Sends a notifications message.
+	SendNotification {
+		/// Name of the protocol for the message.
+		///
+		/// Must match one of the registered protocols. For backwards-compatibility reasons, if
+		/// the remote doesn't support this protocol, we use the legacy substream to send a
+		/// `ConsensusMessage` message.
+		protocol_name: Cow<'static, [u8]>,
+
+		/// The engine ID to use, in case we need to send this message over the legacy substream.
+		///
+		/// > **Note**: Ideally this field wouldn't be necessary, and we would deduce the engine
+		/// >			ID from the existing handlers. However, it is possible (especially in test
+		/// >			situations) that we open connections before all the notification protocols
+		/// >			have been registered, in which case we always rely on the legacy substream.
+		engine_id: ConsensusEngineId,
+
+		/// The message to send.
+		message: Vec<u8>,
+	},
+}
+
+/// Event that can be emitted by a `NotifsHandler`.
+#[derive(Debug)]
+pub enum NotifsHandlerOut {
+	/// Opened the substreams with the remote.
+	Open,
+
+	/// Closed the substreams with the remote.
+	Closed {
+		/// Reason why the substream closed, for diagnostic purposes.
+		reason: Cow<'static, str>,
+	},
+
+	/// Received a non-gossiping message on the legacy substream.
+	CustomMessage {
+		/// Message that has been received.
+		///
+		/// Keep in mind that this can be a `ConsensusMessage` message, which then contains a
+		/// notification.
+		message: BytesMut,
+	},
+
+	/// Received a message on a custom protocol substream.
+	Notification {
+		/// Engine corresponding to the message.
+		protocol_name: Cow<'static, [u8]>,
+
+		/// For legacy reasons, the name to use if we had received the message from the legacy
+		/// substream.
+		engine_id: ConsensusEngineId,
+
+		/// Message that has been received.
+		///
+		/// If `protocol_name` is `None`, this decodes to a `Message`. If `protocol_name` is `Some`,
+		/// this is directly a gossiping message.
+		message: BytesMut,
+	},
+
+	/// A substream to the remote is clogged. The send buffer is very large, and we should print
+	/// a diagnostic message and/or avoid sending more data.
+	Clogged {
+		/// Copy of the messages that are within the buffer, for further diagnostic.
+		messages: Vec<Vec<u8>>,
+	},
+
+	/// An error has happened on the protocol level with this node.
+	ProtocolError {
+		/// If true the error is severe, such as a protocol violation.
+		is_severe: bool,
+		/// The error that happened.
+		error: Box<dyn error::Error + Send + Sync>,
+	},
+}
+
+impl NotifsHandlerProto {
+	/// Builds a new handler.
+	pub fn new(legacy: RegisteredProtocol, list: impl Into<Vec<(Cow<'static, [u8]>, ConsensusEngineId, Vec<u8>)>>) -> Self {
+		let list = list.into();
+
+		NotifsHandlerProto {
+			in_handlers: list.clone().into_iter().map(|(p, e, _)| (NotifsInHandlerProto::new(p), e)).collect(),
+			out_handlers: list.clone().into_iter().map(|(p, e, _)| (NotifsOutHandlerProto::new(p), e)).collect(),
+			legacy: LegacyProtoHandlerProto::new(legacy),
+		}
+	}
+}
+
+impl ProtocolsHandler for NotifsHandler {
+	type InEvent = NotifsHandlerIn;
+	type OutEvent = NotifsHandlerOut;
+	type Error = EitherError<
+		EitherError<
+			<NotifsInHandler as ProtocolsHandler>::Error,
+			<NotifsOutHandler as ProtocolsHandler>::Error,
+		>,
+		<LegacyProtoHandler as ProtocolsHandler>::Error,
+	>;
+	type InboundProtocol = SelectUpgrade<UpgradeCollec<NotificationsIn>, RegisteredProtocol>;
+	type OutboundProtocol = EitherUpgrade<NotificationsOut, RegisteredProtocol>;
+	// Index within the `out_handlers`; None for legacy
+	type OutboundOpenInfo = Option<usize>;
+
+	fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+		let in_handlers = self.in_handlers.iter()
+			.map(|h| h.0.listen_protocol().into_upgrade().1)
+			.collect::<UpgradeCollec<_>>();
+
+		let proto = SelectUpgrade::new(in_handlers, self.legacy.listen_protocol().into_upgrade().1);
+		SubstreamProtocol::new(proto)
+	}
+
+	fn inject_fully_negotiated_inbound(
+		&mut self,
+		out: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output
+	) {
+		match out {
+			EitherOutput::First((out, num)) =>
+				self.in_handlers[num].0.inject_fully_negotiated_inbound(out),
+			EitherOutput::Second(out) =>
+				self.legacy.inject_fully_negotiated_inbound(out),
+		}
+	}
+
+	fn inject_fully_negotiated_outbound(
+		&mut self,
+		out: <Self::OutboundProtocol as OutboundUpgrade<NegotiatedSubstream>>::Output,
+		num: Self::OutboundOpenInfo
+	) {
+		match (out, num) {
+			(EitherOutput::First(out), Some(num)) =>
+				self.out_handlers[num].0.inject_fully_negotiated_outbound(out, ()),
+			(EitherOutput::Second(out), None) =>
+				self.legacy.inject_fully_negotiated_outbound(out, ()),
+			_ => error!("inject_fully_negotiated_outbound called with wrong parameters"),
+		}
+	}
+
+	fn inject_event(&mut self, message: NotifsHandlerIn) {
+		match message {
+			NotifsHandlerIn::Enable => {
+				self.enabled = EnabledState::Enabled;
+				self.legacy.inject_event(LegacyProtoHandlerIn::Enable);
+				for (handler, _) in &mut self.out_handlers {
+					handler.inject_event(NotifsOutHandlerIn::Enable {
+						initial_message: vec![]
+					});
+				}
+				for num in self.pending_in.drain(..) {
+					self.in_handlers[num].0.inject_event(NotifsInHandlerIn::Accept(vec![]));
+				}
+			},
+			NotifsHandlerIn::Disable => {
+				self.legacy.inject_event(LegacyProtoHandlerIn::Disable);
+				// The notifications protocols start in the disabled state. If we were in the
+				// "Initial" state, then we shouldn't disable the notifications protocols again.
+				if self.enabled != EnabledState::Initial {
+					for (handler, _) in &mut self.out_handlers {
+						handler.inject_event(NotifsOutHandlerIn::Disable);
+					}
+				}
+				self.enabled = EnabledState::Disabled;
+				for num in self.pending_in.drain(..) {
+					self.in_handlers[num].0.inject_event(NotifsInHandlerIn::Refuse);
+				}
+			},
+			NotifsHandlerIn::SendLegacy { message } =>
+				self.legacy.inject_event(LegacyProtoHandlerIn::SendCustomMessage { message }),
+			NotifsHandlerIn::SendNotification { message, engine_id, protocol_name } => {
+				for (handler, ngn_id) in &mut self.out_handlers {
+					if handler.protocol_name() != &protocol_name[..] {
+						break;
+					}
+
+					if handler.is_open() {
+						handler.inject_event(NotifsOutHandlerIn::Send(message));
+						return;
+					} else {
+						debug_assert_eq!(engine_id, *ngn_id);
+					}
+				}
+
+				let message = GenericMessage::<(), (), (), ()>::Consensus(ConsensusMessage {
+					engine_id,
+					data: message,
+				});
+
+				self.legacy.inject_event(LegacyProtoHandlerIn::SendCustomMessage {
+					message: message.encode()
+				});
+			},
+		}
+	}
+
+	fn inject_dial_upgrade_error(
+		&mut self,
+		num: Option<usize>,
+		err: ProtocolsHandlerUpgrErr<EitherError<NotificationsHandshakeError, io::Error>>
+	) {
+		match (err, num) {
+			(ProtocolsHandlerUpgrErr::Timeout, Some(num)) =>
+				self.out_handlers[num].0.inject_dial_upgrade_error(
+					(),
+					ProtocolsHandlerUpgrErr::Timeout
+				),
+			(ProtocolsHandlerUpgrErr::Timeout, None) =>
+				self.legacy.inject_dial_upgrade_error((), ProtocolsHandlerUpgrErr::Timeout),
+			(ProtocolsHandlerUpgrErr::Timer, Some(num)) =>
+				self.out_handlers[num].0.inject_dial_upgrade_error(
+					(),
+					ProtocolsHandlerUpgrErr::Timer
+				),
+			(ProtocolsHandlerUpgrErr::Timer, None) =>
+				self.legacy.inject_dial_upgrade_error((), ProtocolsHandlerUpgrErr::Timer),
+			(ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err)), Some(num)) =>
+				self.out_handlers[num].0.inject_dial_upgrade_error(
+					(),
+					ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err))
+				),
+			(ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err)), None) =>
+				self.legacy.inject_dial_upgrade_error(
+					(),
+					ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err))
+				),
+			(ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Apply(EitherError::A(err))), Some(num)) =>
+				self.out_handlers[num].0.inject_dial_upgrade_error(
+					(),
+					ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Apply(err))
+				),
+			(ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Apply(EitherError::B(err))), None) =>
+				self.legacy.inject_dial_upgrade_error(
+					(),
+					ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Apply(err))
+				),
+			_ => error!("inject_dial_upgrade_error called with bad parameters"),
+		}
+	}
+
+	fn connection_keep_alive(&self) -> KeepAlive {
+		// Iterate over each handler and return the maximum value.
+
+		let mut ret = self.legacy.connection_keep_alive();
+		if ret.is_yes() {
+			return KeepAlive::Yes;
+		}
+
+		for (handler, _) in &self.in_handlers {
+			let val = handler.connection_keep_alive();
+			if val.is_yes() {
+				return KeepAlive::Yes;
+			}
+			if ret < val { ret = val; }
+		}
+
+		for (handler, _) in &self.out_handlers {
+			let val = handler.connection_keep_alive();
+			if val.is_yes() {
+				return KeepAlive::Yes;
+			}
+			if ret < val { ret = val; }
+		}
+
+		ret
+	}
+
+	fn poll(
+		&mut self,
+		cx: &mut Context,
+	) -> Poll<
+		ProtocolsHandlerEvent<Self::OutboundProtocol, Self::OutboundOpenInfo, Self::OutEvent, Self::Error>
+	> {
+		for (handler_num, (handler, engine_id)) in self.in_handlers.iter_mut().enumerate() {
+			while let Poll::Ready(ev) = handler.poll(cx) {
+				match ev {
+					ProtocolsHandlerEvent::OutboundSubstreamRequest { .. } =>
+						error!("Incoming substream handler tried to open a substream"),
+					ProtocolsHandlerEvent::Close(err) => void::unreachable(err),
+					ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::OpenRequest(_)) =>
+						match self.enabled {
+							EnabledState::Initial => self.pending_in.push(handler_num),
+							EnabledState::Enabled =>
+								handler.inject_event(NotifsInHandlerIn::Accept(vec![])),
+							EnabledState::Disabled =>
+								handler.inject_event(NotifsInHandlerIn::Refuse),
+						},
+					ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Closed) => {},
+					ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Notif(message)) => {
+						// Note that right now the legacy substream has precedence over
+						// everything. If it is not open, then we consider that nothing is open.
+						if self.legacy.is_open() {
+							let msg = NotifsHandlerOut::Notification {
+								message,
+								engine_id: *engine_id,
+								protocol_name: handler.protocol_name().to_owned().into(),
+							};
+							return Poll::Ready(ProtocolsHandlerEvent::Custom(msg));
+						}
+					},
+				}
+			}
+		}
+
+		for (handler_num, (handler, _)) in self.out_handlers.iter_mut().enumerate() {
+			while let Poll::Ready(ev) = handler.poll(cx) {
+				match ev {
+					ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info: () } =>
+						return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
+							protocol: protocol.map_upgrade(EitherUpgrade::A),
+							info: Some(handler_num),
+						}),
+					ProtocolsHandlerEvent::Close(err) => void::unreachable(err),
+
+					// At the moment we don't actually care whether any notifications protocol
+					// opens or closes.
+					// Whether our communications with the remote are open or closed entirely
+					// depends on the legacy substream, because as long as we are open the user of
+					// this struct might try to send legacy protocol messages which we need to
+					// deliver for things to work properly.
+					ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Open { .. }) => {},
+					ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Closed) => {},
+					ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Refused) => {},
+				}
+			}
+		}
+
+		while let Poll::Ready(ev) = self.legacy.poll(cx) {
+			match ev {
+				ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info: () } =>
+					return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
+						protocol: protocol.map_upgrade(EitherUpgrade::B),
+						info: None,
+					}),
+				ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomProtocolOpen { .. }) =>
+					return Poll::Ready(ProtocolsHandlerEvent::Custom(
+						NotifsHandlerOut::Open
+					)),
+				ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomProtocolClosed { reason }) =>
+					return Poll::Ready(ProtocolsHandlerEvent::Custom(
+						NotifsHandlerOut::Closed { reason }
+					)),
+				ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomMessage { message }) =>
+					return Poll::Ready(ProtocolsHandlerEvent::Custom(
+						NotifsHandlerOut::CustomMessage { message }
+					)),
+				ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::Clogged { messages }) =>
+					return Poll::Ready(ProtocolsHandlerEvent::Custom(
+						NotifsHandlerOut::Clogged { messages }
+					)),
+				ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::ProtocolError { is_severe, error }) =>
+					return Poll::Ready(ProtocolsHandlerEvent::Custom(
+						NotifsHandlerOut::ProtocolError { is_severe, error }
+					)),
+				ProtocolsHandlerEvent::Close(err) =>
+					return Poll::Ready(ProtocolsHandlerEvent::Close(EitherError::B(err))),
+			}
+		}
+
+		Poll::Pending
+	}
+}

substrate/client/network/src/protocol/generic_proto/handler/legacy.rs

@@ -0,0 +1,639 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::protocol::generic_proto::upgrade::{RegisteredProtocol, RegisteredProtocolEvent, RegisteredProtocolSubstream};
+use bytes::BytesMut;
+use futures::prelude::*;
+use futures_timer::Delay;
+use libp2p::core::{ConnectedPoint, PeerId, Endpoint};
+use libp2p::core::upgrade::{InboundUpgrade, OutboundUpgrade};
+use libp2p::swarm::{
+	ProtocolsHandler, ProtocolsHandlerEvent,
+	IntoProtocolsHandler,
+	KeepAlive,
+	ProtocolsHandlerUpgrErr,
+	SubstreamProtocol,
+	NegotiatedSubstream,
+};
+use log::{debug, error};
+use smallvec::{smallvec, SmallVec};
+use std::{borrow::Cow, error, fmt, io, mem, time::Duration};
+use std::{pin::Pin, task::{Context, Poll}};
+
+/// Implements the `IntoProtocolsHandler` trait of libp2p.
+///
+/// Every time a connection with a remote starts, an instance of this struct is created and
+/// sent to a background task dedicated to this connection. Once the connection is established,
+/// it is turned into a `LegacyProtoHandler`. It then handles all communications that are specific
+/// to Substrate on that single connection.
+///
+/// Note that there can be multiple instance of this struct simultaneously for same peer. However
+/// if that happens, only one main instance can communicate with the outer layers of the code. In
+/// other words, the outer layers of the code only ever see one handler.
+///
+/// ## State of the handler
+///
+/// There are six possible states for the handler:
+///
+/// - Enabled and open, which is a normal operation.
+/// - Enabled and closed, in which case it will try to open substreams.
+/// - Disabled and open, in which case it will try to close substreams.
+/// - Disabled and closed, in which case the handler is idle. The connection will be
+///   garbage-collected after a few seconds if nothing more happens.
+/// - Initializing and open.
+/// - Initializing and closed, which is the state the handler starts in.
+///
+/// The Init/Enabled/Disabled state is entirely controlled by the user by sending `Enable` or
+/// `Disable` messages to the handler. The handler itself never transitions automatically between
+/// these states. For example, if the handler reports a network misbehaviour, it will close the
+/// substreams but it is the role of the user to send a `Disabled` event if it wants the connection
+/// to close. Otherwise, the handler will try to reopen substreams.
+/// The handler starts in the "Initializing" state and must be transitionned to Enabled or Disabled
+/// as soon as possible.
+///
+/// The Open/Closed state is decided by the handler and is reported with the `CustomProtocolOpen`
+/// and `CustomProtocolClosed` events. The `CustomMessage` event can only be generated if the
+/// handler is open.
+///
+/// ## How it works
+///
+/// When the handler is created, it is initially in the `Init` state and waits for either a
+/// `Disable` or an `Enable` message from the outer layer. At any time, the outer layer is free to
+/// toggle the handler between the disabled and enabled states.
+///
+/// When the handler switches to "enabled", it opens a substream and negotiates the protocol named
+/// `/substrate/xxx`, where `xxx` is chosen by the user and depends on the chain.
+///
+/// For backwards compatibility reasons, when we switch to "enabled" for the first time (while we
+/// are still in "init" mode) and we are the connection listener, we don't open a substream.
+///
+/// In order the handle the situation where both the remote and us get enabled at the same time,
+/// we tolerate multiple substreams open at the same time. Messages are transmitted on an arbitrary
+/// substream. The endpoints don't try to agree on a single substream.
+///
+/// We consider that we are now "closed" if the remote closes all the existing substreams.
+/// Re-opening it can then be performed by closing all active substream and re-opening one.
+///
+pub struct LegacyProtoHandlerProto {
+	/// Configuration for the protocol upgrade to negotiate.
+	protocol: RegisteredProtocol,
+}
+
+impl LegacyProtoHandlerProto {
+	/// Builds a new `LegacyProtoHandlerProto`.
+	pub fn new(protocol: RegisteredProtocol) -> Self {
+		LegacyProtoHandlerProto {
+			protocol,
+		}
+	}
+}
+
+impl IntoProtocolsHandler for LegacyProtoHandlerProto {
+	type Handler = LegacyProtoHandler;
+
+	fn inbound_protocol(&self) -> RegisteredProtocol {
+		self.protocol.clone()
+	}
+
+	fn into_handler(self, remote_peer_id: &PeerId, connected_point: &ConnectedPoint) -> Self::Handler {
+		LegacyProtoHandler {
+			protocol: self.protocol,
+			endpoint: connected_point.to_endpoint(),
+			remote_peer_id: remote_peer_id.clone(),
+			state: ProtocolState::Init {
+				substreams: SmallVec::new(),
+				init_deadline: Delay::new(Duration::from_secs(5))
+			},
+			events_queue: SmallVec::new(),
+		}
+	}
+}
+
+/// The actual handler once the connection has been established.
+pub struct LegacyProtoHandler {
+	/// Configuration for the protocol upgrade to negotiate.
+	protocol: RegisteredProtocol,
+
+	/// State of the communications with the remote.
+	state: ProtocolState,
+
+	/// Identifier of the node we're talking to. Used only for logging purposes and shouldn't have
+	/// any influence on the behaviour.
+	remote_peer_id: PeerId,
+
+	/// Whether we are the connection dialer or listener. Used to determine who, between the local
+	/// node and the remote node, has priority.
+	endpoint: Endpoint,
+
+	/// Queue of events to send to the outside.
+	///
+	/// This queue must only ever be modified to insert elements at the back, or remove the first
+	/// element.
+	events_queue: SmallVec<[ProtocolsHandlerEvent<RegisteredProtocol, (), LegacyProtoHandlerOut, ConnectionKillError>; 16]>,
+}
+
+/// State of the handler.
+enum ProtocolState {
+	/// Waiting for the behaviour to tell the handler whether it is enabled or disabled.
+	Init {
+		/// List of substreams opened by the remote but that haven't been processed yet.
+		substreams: SmallVec<[RegisteredProtocolSubstream<NegotiatedSubstream>; 6]>,
+		/// Deadline after which the initialization is abnormally long.
+		init_deadline: Delay,
+	},
+
+	/// Handler is opening a substream in order to activate itself.
+	/// If we are in this state, we haven't sent any `CustomProtocolOpen` yet.
+	Opening {
+		/// Deadline after which the opening is abnormally long.
+		deadline: Delay,
+	},
+
+	/// Normal operating mode. Contains the substreams that are open.
+	/// If we are in this state, we have sent a `CustomProtocolOpen` message to the outside.
+	Normal {
+		/// The substreams where bidirectional communications happen.
+		substreams: SmallVec<[RegisteredProtocolSubstream<NegotiatedSubstream>; 4]>,
+		/// Contains substreams which are being shut down.
+		shutdown: SmallVec<[RegisteredProtocolSubstream<NegotiatedSubstream>; 4]>,
+	},
+
+	/// We are disabled. Contains substreams that are being closed.
+	/// If we are in this state, either we have sent a `CustomProtocolClosed` message to the
+	/// outside or we have never sent any `CustomProtocolOpen` in the first place.
+	Disabled {
+		/// List of substreams to shut down.
+		shutdown: SmallVec<[RegisteredProtocolSubstream<NegotiatedSubstream>; 6]>,
+
+		/// If true, we should reactivate the handler after all the substreams in `shutdown` have
+		/// been closed.
+		///
+		/// Since we don't want to mix old and new substreams, we wait for all old substreams to
+		/// be closed before opening any new one.
+		reenable: bool,
+	},
+
+	/// In this state, we don't care about anything anymore and need to kill the connection as soon
+	/// as possible.
+	KillAsap,
+
+	/// We sometimes temporarily switch to this state during processing. If we are in this state
+	/// at the beginning of a method, that means something bad happened in the source code.
+	Poisoned,
+}
+
+/// Event that can be received by a `LegacyProtoHandler`.
+#[derive(Debug)]
+pub enum LegacyProtoHandlerIn {
+	/// The node should start using custom protocols.
+	Enable,
+
+	/// The node should stop using custom protocols.
+	Disable,
+
+	/// Sends a message through a custom protocol substream.
+	SendCustomMessage {
+		/// The message to send.
+		message: Vec<u8>,
+	},
+}
+
+/// Event that can be emitted by a `LegacyProtoHandler`.
+#[derive(Debug)]
+pub enum LegacyProtoHandlerOut {
+	/// Opened a custom protocol with the remote.
+	CustomProtocolOpen {
+		/// Version of the protocol that has been opened.
+		version: u8,
+	},
+
+	/// Closed a custom protocol with the remote.
+	CustomProtocolClosed {
+		/// Reason why the substream closed, for diagnostic purposes.
+		reason: Cow<'static, str>,
+	},
+
+	/// Receives a message on a custom protocol substream.
+	CustomMessage {
+		/// Message that has been received.
+		message: BytesMut,
+	},
+
+	/// A substream to the remote is clogged. The send buffer is very large, and we should print
+	/// a diagnostic message and/or avoid sending more data.
+	Clogged {
+		/// Copy of the messages that are within the buffer, for further diagnostic.
+		messages: Vec<Vec<u8>>,
+	},
+
+	/// An error has happened on the protocol level with this node.
+	ProtocolError {
+		/// If true the error is severe, such as a protocol violation.
+		is_severe: bool,
+		/// The error that happened.
+		error: Box<dyn error::Error + Send + Sync>,
+	},
+}
+
+impl LegacyProtoHandler {
+	/// Returns true if the legacy substream is currently open.
+	pub fn is_open(&self) -> bool {
+		match &self.state {
+			ProtocolState::Init { substreams, .. } => !substreams.is_empty(),
+			ProtocolState::Opening { .. } => false,
+			ProtocolState::Normal { substreams, .. } => !substreams.is_empty(),
+			ProtocolState::Disabled { .. } => false,
+			ProtocolState::KillAsap => false,
+			ProtocolState::Poisoned => false,
+		}
+	}
+
+	/// Enables the handler.
+	fn enable(&mut self) {
+		self.state = match mem::replace(&mut self.state, ProtocolState::Poisoned) {
+			ProtocolState::Poisoned => {
+				error!(target: "sub-libp2p", "Handler with {:?} is in poisoned state",
+					self.remote_peer_id);
+				ProtocolState::Poisoned
+			}
+
+			ProtocolState::Init { substreams: incoming, .. } => {
+				if incoming.is_empty() {
+					if let Endpoint::Dialer = self.endpoint {
+						self.events_queue.push(ProtocolsHandlerEvent::OutboundSubstreamRequest {
+							protocol: SubstreamProtocol::new(self.protocol.clone()),
+							info: (),
+						});
+					}
+					ProtocolState::Opening {
+						deadline: Delay::new(Duration::from_secs(60))
+					}
+
+				} else {
+					let event = LegacyProtoHandlerOut::CustomProtocolOpen {
+						version: incoming[0].protocol_version()
+					};
+					self.events_queue.push(ProtocolsHandlerEvent::Custom(event));
+					ProtocolState::Normal {
+						substreams: incoming.into_iter().collect(),
+						shutdown: SmallVec::new()
+					}
+				}
+			}
+
+			st @ ProtocolState::KillAsap => st,
+			st @ ProtocolState::Opening { .. } => st,
+			st @ ProtocolState::Normal { .. } => st,
+			ProtocolState::Disabled { shutdown, .. } => {
+				ProtocolState::Disabled { shutdown, reenable: true }
+			}
+		}
+	}
+
+	/// Disables the handler.
+	fn disable(&mut self) {
+		self.state = match mem::replace(&mut self.state, ProtocolState::Poisoned) {
+			ProtocolState::Poisoned => {
+				error!(target: "sub-libp2p", "Handler with {:?} is in poisoned state",
+					self.remote_peer_id);
+				ProtocolState::Poisoned
+			}
+
+			ProtocolState::Init { substreams: mut shutdown, .. } => {
+				for s in &mut shutdown {
+					s.shutdown();
+				}
+				ProtocolState::Disabled { shutdown, reenable: false }
+			}
+
+			ProtocolState::Opening { .. } | ProtocolState::Normal { .. } =>
+				// At the moment, if we get disabled while things were working, we kill the entire
+				// connection in order to force a reset of the state.
+				// This is obviously an extremely shameful way to do things, but at the time of
+				// the writing of this comment, the networking works very poorly and a solution
+				// needs to be found.
+				ProtocolState::KillAsap,
+
+			ProtocolState::Disabled { shutdown, .. } =>
+				ProtocolState::Disabled { shutdown, reenable: false },
+
+			ProtocolState::KillAsap => ProtocolState::KillAsap,
+		};
+	}
+
+	/// Polls the state for events. Optionally returns an event to produce.
+	#[must_use]
+	fn poll_state(&mut self, cx: &mut Context)
+		-> Option<ProtocolsHandlerEvent<RegisteredProtocol, (), LegacyProtoHandlerOut, ConnectionKillError>> {
+		match mem::replace(&mut self.state, ProtocolState::Poisoned) {
+			ProtocolState::Poisoned => {
+				error!(target: "sub-libp2p", "Handler with {:?} is in poisoned state",
+					self.remote_peer_id);
+				self.state = ProtocolState::Poisoned;
+				None
+			}
+
+			ProtocolState::Init { substreams, mut init_deadline } => {
+				match Pin::new(&mut init_deadline).poll(cx) {
+					Poll::Ready(()) => {
+						init_deadline = Delay::new(Duration::from_secs(60));
+						error!(target: "sub-libp2p", "Handler initialization process is too long \
+							with {:?}", self.remote_peer_id)
+					},
+					Poll::Pending => {}
+				}
+
+				self.state = ProtocolState::Init { substreams, init_deadline };
+				None
+			}
+
+			ProtocolState::Opening { mut deadline } => {
+				match Pin::new(&mut deadline).poll(cx) {
+					Poll::Ready(()) => {
+						deadline = Delay::new(Duration::from_secs(60));
+						let event = LegacyProtoHandlerOut::ProtocolError {
+							is_severe: true,
+							error: "Timeout when opening protocol".to_string().into(),
+						};
+						self.state = ProtocolState::Opening { deadline };
+						Some(ProtocolsHandlerEvent::Custom(event))
+					},
+					Poll::Pending => {
+						self.state = ProtocolState::Opening { deadline };
+						None
+					},
+				}
+			}
+
+			ProtocolState::Normal { mut substreams, mut shutdown } => {
+				for n in (0..substreams.len()).rev() {
+					let mut substream = substreams.swap_remove(n);
+					match Pin::new(&mut substream).poll_next(cx) {
+						Poll::Pending => substreams.push(substream),
+						Poll::Ready(Some(Ok(RegisteredProtocolEvent::Message(message)))) => {
+							let event = LegacyProtoHandlerOut::CustomMessage {
+								message
+							};
+							substreams.push(substream);
+							self.state = ProtocolState::Normal { substreams, shutdown };
+							return Some(ProtocolsHandlerEvent::Custom(event));
+						},
+						Poll::Ready(Some(Ok(RegisteredProtocolEvent::Clogged { messages }))) => {
+							let event = LegacyProtoHandlerOut::Clogged {
+								messages,
+							};
+							substreams.push(substream);
+							self.state = ProtocolState::Normal { substreams, shutdown };
+							return Some(ProtocolsHandlerEvent::Custom(event));
+						}
+						Poll::Ready(None) => {
+							shutdown.push(substream);
+							if substreams.is_empty() {
+								let event = LegacyProtoHandlerOut::CustomProtocolClosed {
+									reason: "All substreams have been closed by the remote".into(),
+								};
+								self.state = ProtocolState::Disabled {
+									shutdown: shutdown.into_iter().collect(),
+									reenable: true
+								};
+								return Some(ProtocolsHandlerEvent::Custom(event));
+							}
+						}
+						Poll::Ready(Some(Err(err))) => {
+							if substreams.is_empty() {
+								let event = LegacyProtoHandlerOut::CustomProtocolClosed {
+									reason: format!("Error on the last substream: {:?}", err).into(),
+								};
+								self.state = ProtocolState::Disabled {
+									shutdown: shutdown.into_iter().collect(),
+									reenable: true
+								};
+								return Some(ProtocolsHandlerEvent::Custom(event));
+							} else {
+								debug!(target: "sub-libp2p", "Error on extra substream: {:?}", err);
+							}
+						}
+					}
+				}
+
+				// This code is reached is none if and only if none of the substreams are in a ready state.
+				self.state = ProtocolState::Normal { substreams, shutdown };
+				None
+			}
+
+			ProtocolState::Disabled { mut shutdown, reenable } => {
+				shutdown_list(&mut shutdown, cx);
+				// If `reenable` is `true`, that means we should open the substreams system again
+				// after all the substreams are closed.
+				if reenable && shutdown.is_empty() {
+					self.state = ProtocolState::Opening {
+						deadline: Delay::new(Duration::from_secs(60))
+					};
+					Some(ProtocolsHandlerEvent::OutboundSubstreamRequest {
+						protocol: SubstreamProtocol::new(self.protocol.clone()),
+						info: (),
+					})
+				} else {
+					self.state = ProtocolState::Disabled { shutdown, reenable };
+					None
+				}
+			}
+
+			ProtocolState::KillAsap => None,
+		}
+	}
+
+	/// Called by `inject_fully_negotiated_inbound` and `inject_fully_negotiated_outbound`.
+	fn inject_fully_negotiated(
+		&mut self,
+		mut substream: RegisteredProtocolSubstream<NegotiatedSubstream>
+	) {
+		self.state = match mem::replace(&mut self.state, ProtocolState::Poisoned) {
+			ProtocolState::Poisoned => {
+				error!(target: "sub-libp2p", "Handler with {:?} is in poisoned state",
+					self.remote_peer_id);
+				ProtocolState::Poisoned
+			}
+
+			ProtocolState::Init { mut substreams, init_deadline } => {
+				if substream.endpoint() == Endpoint::Dialer {
+					error!(target: "sub-libp2p", "Opened dialing substream with {:?} before \
+						initialization", self.remote_peer_id);
+				}
+				substreams.push(substream);
+				ProtocolState::Init { substreams, init_deadline }
+			}
+
+			ProtocolState::Opening { .. } => {
+				let event = LegacyProtoHandlerOut::CustomProtocolOpen {
+					version: substream.protocol_version()
+				};
+				self.events_queue.push(ProtocolsHandlerEvent::Custom(event));
+				ProtocolState::Normal {
+					substreams: smallvec![substream],
+					shutdown: SmallVec::new()
+				}
+			}
+
+			ProtocolState::Normal { substreams: mut existing, shutdown } => {
+				existing.push(substream);
+				ProtocolState::Normal { substreams: existing, shutdown }
+			}
+
+			ProtocolState::Disabled { mut shutdown, .. } => {
+				substream.shutdown();
+				shutdown.push(substream);
+				ProtocolState::Disabled { shutdown, reenable: false }
+			}
+
+			ProtocolState::KillAsap => ProtocolState::KillAsap,
+		};
+	}
+
+	/// Sends a message to the remote.
+	fn send_message(&mut self, message: Vec<u8>) {
+		match self.state {
+			ProtocolState::Normal { ref mut substreams, .. } =>
+				substreams[0].send_message(message),
+
+			_ => debug!(target: "sub-libp2p", "Tried to send message over closed protocol \
+				with {:?}", self.remote_peer_id)
+		}
+	}
+}
+
+impl ProtocolsHandler for LegacyProtoHandler {
+	type InEvent = LegacyProtoHandlerIn;
+	type OutEvent = LegacyProtoHandlerOut;
+	type Error = ConnectionKillError;
+	type InboundProtocol = RegisteredProtocol;
+	type OutboundProtocol = RegisteredProtocol;
+	type OutboundOpenInfo = ();
+
+	fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+		SubstreamProtocol::new(self.protocol.clone())
+	}
+
+	fn inject_fully_negotiated_inbound(
+		&mut self,
+		proto: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output
+	) {
+		self.inject_fully_negotiated(proto);
+	}
+
+	fn inject_fully_negotiated_outbound(
+		&mut self,
+		proto: <Self::OutboundProtocol as OutboundUpgrade<NegotiatedSubstream>>::Output,
+		_: Self::OutboundOpenInfo
+	) {
+		self.inject_fully_negotiated(proto);
+	}
+
+	fn inject_event(&mut self, message: LegacyProtoHandlerIn) {
+		match message {
+			LegacyProtoHandlerIn::Disable => self.disable(),
+			LegacyProtoHandlerIn::Enable => self.enable(),
+			LegacyProtoHandlerIn::SendCustomMessage { message } =>
+				self.send_message(message),
+		}
+	}
+
+	#[inline]
+	fn inject_dial_upgrade_error(&mut self, _: (), err: ProtocolsHandlerUpgrErr<io::Error>) {
+		let is_severe = match err {
+			ProtocolsHandlerUpgrErr::Upgrade(_) => true,
+			_ => false,
+		};
+
+		self.events_queue.push(ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::ProtocolError {
+			is_severe,
+			error: Box::new(err),
+		}));
+	}
+
+	fn connection_keep_alive(&self) -> KeepAlive {
+		match self.state {
+			ProtocolState::Init { .. } | ProtocolState::Opening { .. } |
+			ProtocolState::Normal { .. } => KeepAlive::Yes,
+			ProtocolState::Disabled { .. } | ProtocolState::Poisoned |
+	  		ProtocolState::KillAsap => KeepAlive::No,
+		}
+	}
+
+	fn poll(
+		&mut self,
+		cx: &mut Context,
+	) -> Poll<
+		ProtocolsHandlerEvent<Self::OutboundProtocol, Self::OutboundOpenInfo, Self::OutEvent, Self::Error>
+	> {
+		// Flush the events queue if necessary.
+		if !self.events_queue.is_empty() {
+			let event = self.events_queue.remove(0);
+			return Poll::Ready(event)
+		}
+
+		// Kill the connection if needed.
+		if let ProtocolState::KillAsap = self.state {
+			return Poll::Ready(ProtocolsHandlerEvent::Close(ConnectionKillError));
+		}
+
+		// Process all the substreams.
+		if let Some(event) = self.poll_state(cx) {
+			return Poll::Ready(event)
+		}
+
+		Poll::Pending
+	}
+}
+
+impl fmt::Debug for LegacyProtoHandler {
+	fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+		f.debug_struct("LegacyProtoHandler")
+			.finish()
+	}
+}
+
+/// Given a list of substreams, tries to shut them down. The substreams that have been successfully
+/// shut down are removed from the list.
+fn shutdown_list
+	(list: &mut SmallVec<impl smallvec::Array<Item = RegisteredProtocolSubstream<NegotiatedSubstream>>>,
+	cx: &mut Context)
+{
+	'outer: for n in (0..list.len()).rev() {
+		let mut substream = list.swap_remove(n);
+		loop {
+			match substream.poll_next_unpin(cx) {
+				Poll::Ready(Some(Ok(_))) => {}
+				Poll::Pending => break,
+				Poll::Ready(Some(Err(_))) | Poll::Ready(None) => continue 'outer,
+			}
+		}
+		list.push(substream);
+	}
+}
+
+/// Error returned when switching from normal to disabled.
+#[derive(Debug)]
+pub struct ConnectionKillError;
+
+impl error::Error for ConnectionKillError {
+}
+
+impl fmt::Display for ConnectionKillError {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		write!(f, "Connection kill when switching from normal to disabled")
+	}
+}

substrate/client/network/src/protocol/generic_proto/handler/notif_in.rs

@@ -0,0 +1,256 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Implementations of the `IntoProtocolsHandler` and `ProtocolsHandler` traits for ingoing
+//! substreams for a single gossiping protocol.
+//!
+//! > **Note**: Each instance corresponds to a single protocol. In order to support multiple
+//! >			protocols, you need to create multiple instances and group them.
+//!
+
+use crate::protocol::generic_proto::upgrade::{NotificationsIn, NotificationsInSubstream};
+use bytes::BytesMut;
+use futures::prelude::*;
+use libp2p::core::{ConnectedPoint, PeerId};
+use libp2p::core::upgrade::{DeniedUpgrade, InboundUpgrade, OutboundUpgrade};
+use libp2p::swarm::{
+	ProtocolsHandler, ProtocolsHandlerEvent,
+	IntoProtocolsHandler,
+	KeepAlive,
+	ProtocolsHandlerUpgrErr,
+	SubstreamProtocol,
+	NegotiatedSubstream,
+};
+use log::{error, warn};
+use smallvec::SmallVec;
+use std::{borrow::Cow, fmt, pin::Pin, str, task::{Context, Poll}};
+
+/// Implements the `IntoProtocolsHandler` trait of libp2p.
+///
+/// Every time a connection with a remote starts, an instance of this struct is created and
+/// sent to a background task dedicated to this connection. Once the connection is established,
+/// it is turned into a [`NotifsInHandler`].
+pub struct NotifsInHandlerProto {
+	/// Configuration for the protocol upgrade to negotiate.
+	in_protocol: NotificationsIn,
+}
+
+/// The actual handler once the connection has been established.
+pub struct NotifsInHandler {
+	/// Configuration for the protocol upgrade to negotiate for inbound substreams.
+	in_protocol: NotificationsIn,
+
+	/// Substream that is open with the remote.
+	substream: Option<NotificationsInSubstream<NegotiatedSubstream>>,
+
+	/// If the substream is opened and closed rapidly, we can emit several `OpenRequest` and
+	/// `Closed` messages in a row without the handler having time to respond with `Accept` or
+	/// `Refuse`.
+	///
+	/// In order to keep the state consistent, we increment this variable every time an
+	/// `OpenRequest` is emitted and decrement it every time an `Accept` or `Refuse` is received.
+	pending_accept_refuses: usize,
+
+	/// Queue of events to send to the outside.
+	///
+	/// This queue is only ever modified to insert elements at the back, or remove the first
+	/// element.
+	events_queue: SmallVec<[ProtocolsHandlerEvent<DeniedUpgrade, (), NotifsInHandlerOut, void::Void>; 16]>,
+}
+
+/// Event that can be received by a `NotifsInHandler`.
+#[derive(Debug)]
+pub enum NotifsInHandlerIn {
+	/// Can be sent back as a response to an `OpenRequest`. Contains the status message to send
+	/// to the remote.
+	///
+	/// After sending this to the handler, the substream is now considered open and `Notif` events
+	/// can be received.
+	Accept(Vec<u8>),
+
+	/// Can be sent back as a response to an `OpenRequest`.
+	Refuse,
+}
+
+/// Event that can be emitted by a `NotifsInHandler`.
+#[derive(Debug)]
+pub enum NotifsInHandlerOut {
+	/// The remote wants to open a substream. Contains the initial message sent by the remote
+	/// when the substream has been opened.
+	///
+	/// Every time this event is emitted, a corresponding `Accepted` or `Refused` **must** be sent
+	/// back even if a `Closed` is received.
+	OpenRequest(Vec<u8>),
+
+	/// The notifications substream has been closed by the remote. In order to avoid race
+	/// conditions, this does **not** cancel any previously-sent `OpenRequest`.
+	Closed,
+
+	/// Received a message on the notifications substream.
+	///
+	/// Can only happen after an `Accept` and before a `Closed`.
+	Notif(BytesMut),
+}
+
+impl NotifsInHandlerProto {
+	/// Builds a new `NotifsInHandlerProto`.
+	pub fn new(
+		protocol_name: impl Into<Cow<'static, [u8]>>
+	) -> Self {
+		NotifsInHandlerProto {
+			in_protocol: NotificationsIn::new(protocol_name),
+		}
+	}
+}
+
+impl IntoProtocolsHandler for NotifsInHandlerProto {
+	type Handler = NotifsInHandler;
+
+	fn inbound_protocol(&self) -> NotificationsIn {
+		self.in_protocol.clone()
+	}
+
+	fn into_handler(self, _: &PeerId, _: &ConnectedPoint) -> Self::Handler {
+		NotifsInHandler {
+			in_protocol: self.in_protocol,
+			substream: None,
+			pending_accept_refuses: 0,
+			events_queue: SmallVec::new(),
+		}
+	}
+}
+
+impl NotifsInHandler {
+	/// Returns the name of the protocol that we accept.
+	pub fn protocol_name(&self) -> &[u8] {
+		self.in_protocol.protocol_name()
+	}
+}
+
+impl ProtocolsHandler for NotifsInHandler {
+	type InEvent = NotifsInHandlerIn;
+	type OutEvent = NotifsInHandlerOut;
+	type Error = void::Void;
+	type InboundProtocol = NotificationsIn;
+	type OutboundProtocol = DeniedUpgrade;
+	type OutboundOpenInfo = ();
+
+	fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+		SubstreamProtocol::new(self.in_protocol.clone())
+	}
+
+	fn inject_fully_negotiated_inbound(
+		&mut self,
+		(msg, proto): <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output
+	) {
+		if self.substream.is_some() {
+			warn!(
+				target: "sub-libp2p",
+				"Received duplicate inbound notifications substream for {:?}",
+				str::from_utf8(self.in_protocol.protocol_name()),
+			);
+			return;
+		}
+
+		self.substream = Some(proto);
+		self.events_queue.push(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::OpenRequest(msg)));
+		self.pending_accept_refuses = self.pending_accept_refuses
+			.checked_add(1)
+			.unwrap_or_else(|| {
+				error!(target: "sub-libp2p", "Overflow in pending_accept_refuses");
+				usize::max_value()
+			});
+	}
+
+	fn inject_fully_negotiated_outbound(
+		&mut self,
+		out: <Self::OutboundProtocol as OutboundUpgrade<NegotiatedSubstream>>::Output,
+		_: Self::OutboundOpenInfo
+	) {
+		// We never emit any outgoing substream.
+		void::unreachable(out)
+	}
+
+	fn inject_event(&mut self, message: NotifsInHandlerIn) {
+		self.pending_accept_refuses = match self.pending_accept_refuses.checked_sub(1) {
+			Some(v) => v,
+			None => {
+				error!(
+					target: "sub-libp2p",
+					"Inconsistent state: received Accept/Refuse when no pending request exists"
+				);
+				return;
+			}
+		};
+
+		// If we send multiple `OpenRequest`s in a row, we will receive back multiple
+		// `Accept`/`Refuse` messages. All of them are obsolete except the last one.
+		if self.pending_accept_refuses != 0 {
+			return;
+		}
+
+		match (message, self.substream.as_mut()) {
+			(NotifsInHandlerIn::Accept(message), Some(sub)) => sub.send_handshake(message),
+			(NotifsInHandlerIn::Accept(_), None) => {},
+			(NotifsInHandlerIn::Refuse, _) => self.substream = None,
+		}
+	}
+
+	fn inject_dial_upgrade_error(&mut self, _: (), _: ProtocolsHandlerUpgrErr<void::Void>) {
+		error!(target: "sub-libp2p", "Received dial upgrade error in inbound-only handler");
+	}
+
+	fn connection_keep_alive(&self) -> KeepAlive {
+		if self.substream.is_some() {
+			KeepAlive::Yes
+		} else {
+			KeepAlive::No
+		}
+	}
+
+	fn poll(
+		&mut self,
+		cx: &mut Context,
+	) -> Poll<
+		ProtocolsHandlerEvent<Self::OutboundProtocol, Self::OutboundOpenInfo, Self::OutEvent, Self::Error>
+	> {
+		// Flush the events queue if necessary.
+		if !self.events_queue.is_empty() {
+			let event = self.events_queue.remove(0);
+			return Poll::Ready(event)
+		}
+
+		match self.substream.as_mut().map(|s| Stream::poll_next(Pin::new(s), cx)) {
+			None | Some(Poll::Pending) => {},
+			Some(Poll::Ready(Some(Ok(msg)))) =>
+				return Poll::Ready(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Notif(msg))),
+			Some(Poll::Ready(None)) | Some(Poll::Ready(Some(Err(_)))) => {
+				self.substream = None;
+				return Poll::Ready(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Closed));
+			},
+		}
+
+		Poll::Pending
+	}
+}
+
+impl fmt::Debug for NotifsInHandler {
+	fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+		f.debug_struct("NotifsInHandler")
+			.field("substream_open", &self.substream.is_some())
+			.finish()
+	}
+}

substrate/client/network/src/protocol/generic_proto/handler/notif_out.rs

@@ -0,0 +1,395 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Implementations of the `IntoProtocolsHandler` and `ProtocolsHandler` traits for outgoing
+//! substreams of a single gossiping protocol.
+//!
+//! > **Note**: Each instance corresponds to a single protocol. In order to support multiple
+//! >			protocols, you need to create multiple instances and group them.
+//!
+
+use crate::protocol::generic_proto::upgrade::{NotificationsOut, NotificationsOutSubstream, NotificationsHandshakeError};
+use futures::prelude::*;
+use libp2p::core::{ConnectedPoint, PeerId};
+use libp2p::core::upgrade::{DeniedUpgrade, InboundUpgrade, OutboundUpgrade};
+use libp2p::swarm::{
+	ProtocolsHandler, ProtocolsHandlerEvent,
+	IntoProtocolsHandler,
+	KeepAlive,
+	ProtocolsHandlerUpgrErr,
+	SubstreamProtocol,
+	NegotiatedSubstream,
+};
+use log::error;
+use smallvec::SmallVec;
+use std::{borrow::Cow, fmt, mem, pin::Pin, task::{Context, Poll}, time::Duration};
+use wasm_timer::Instant;
+
+/// Maximum duration to open a substream and receive the handshake message. After that, we
+/// consider that we failed to open the substream.
+const OPEN_TIMEOUT: Duration = Duration::from_secs(10);
+/// After successfully establishing a connection with the remote, we keep the connection open for
+/// at least this amount of time in order to give the rest of the code the chance to notify us to
+/// open substreams.
+const INITIAL_KEEPALIVE_TIME: Duration = Duration::from_secs(5);
+
+/// Implements the `IntoProtocolsHandler` trait of libp2p.
+///
+/// Every time a connection with a remote starts, an instance of this struct is created and
+/// sent to a background task dedicated to this connection. Once the connection is established,
+/// it is turned into a [`NotifsOutHandler`].
+///
+/// See the documentation of [`NotifsOutHandler`] for more information.
+pub struct NotifsOutHandlerProto {
+	/// Name of the protocol to negotiate.
+	protocol_name: Cow<'static, [u8]>,
+}
+
+impl NotifsOutHandlerProto {
+	/// Builds a new [`NotifsOutHandlerProto`]. Will use the given protocol name for the
+	/// notifications substream.
+	pub fn new(protocol_name: impl Into<Cow<'static, [u8]>>) -> Self {
+		NotifsOutHandlerProto {
+			protocol_name: protocol_name.into(),
+		}
+	}
+}
+
+impl IntoProtocolsHandler for NotifsOutHandlerProto {
+	type Handler = NotifsOutHandler;
+
+	fn inbound_protocol(&self) -> DeniedUpgrade {
+		DeniedUpgrade
+	}
+
+	fn into_handler(self, _: &PeerId, _: &ConnectedPoint) -> Self::Handler {
+		NotifsOutHandler {
+			protocol_name: self.protocol_name,
+			when_connection_open: Instant::now(),
+			state: State::Disabled,
+			events_queue: SmallVec::new(),
+		}
+	}
+}
+
+/// Handler for an outbound notification substream.
+///
+/// When a connection is established, this handler starts in the "disabled" state, meaning that
+/// no substream will be open.
+///
+/// One can try open a substream by sending an [`NotifsOutHandlerIn::Enable`] message to the
+/// handler. Once done, the handler will try to establish then maintain an outbound substream with
+/// the remote for the purpose of sending notifications to it.
+pub struct NotifsOutHandler {
+	/// Name of the protocol to negotiate.
+	protocol_name: Cow<'static, [u8]>,
+
+	/// Relationship with the node we're connected to.
+	state: State,
+
+	/// When the connection with the remote has been successfully established.
+	when_connection_open: Instant,
+
+	/// Queue of events to send to the outside.
+	///
+	/// This queue must only ever be modified to insert elements at the back, or remove the first
+	/// element.
+	events_queue: SmallVec<[ProtocolsHandlerEvent<NotificationsOut, (), NotifsOutHandlerOut, void::Void>; 16]>,
+}
+
+/// Our relationship with the node we're connected to.
+enum State {
+	/// The handler is disabled and idle. No substream is open.
+	Disabled,
+
+	/// The handler is disabled. A substream is still open and needs to be closed.
+	///
+	/// > **Important**: Having this state means that `poll_close` has been called at least once,
+	/// >				 but the `Sink` API is unclear about whether or not the stream can then
+	/// >				 be recovered. Because of that, we must never switch from the
+	/// >				 `DisabledOpen` state to the `Open` state while keeping the same substream.
+	DisabledOpen(NotificationsOutSubstream<NegotiatedSubstream>),
+
+	/// The handler is disabled but we are still trying to open a substream with the remote.
+	///
+	/// If the handler gets enabled again, we can immediately switch to `Opening`.
+	DisabledOpening,
+
+	/// The handler is enabled and we are trying to open a substream with the remote.
+	Opening {
+		/// The initial message that we sent. Necessary if we need to re-open a substream.
+		initial_message: Vec<u8>,
+	},
+
+	/// The handler is enabled. We have tried opening a substream in the past but the remote
+	/// refused it.
+	Refused,
+
+	/// The handler is enabled and substream is open.
+	Open {
+		/// Substream that is currently open.
+		substream: NotificationsOutSubstream<NegotiatedSubstream>,
+		/// The initial message that we sent. Necessary if we need to re-open a substream.
+		initial_message: Vec<u8>,
+	},
+
+	/// Poisoned state. Shouldn't be found in the wild.
+	Poisoned,
+}
+
+/// Event that can be received by a `NotifsOutHandler`.
+#[derive(Debug)]
+pub enum NotifsOutHandlerIn {
+	/// Enables the notifications substream for this node. The handler will try to maintain a
+	/// substream with the remote.
+	Enable {
+		/// Initial message to send to remote nodes when we open substreams.
+		initial_message: Vec<u8>,
+	},
+
+	/// Disables the notifications substream for this node. This is the default state.
+	Disable,
+
+	/// Sends a message on the notifications substream. Ignored if the substream isn't open.
+	///
+	/// It is only valid to send this if the notifications substream has been enabled.
+	Send(Vec<u8>),
+}
+
+/// Event that can be emitted by a `NotifsOutHandler`.
+#[derive(Debug)]
+pub enum NotifsOutHandlerOut {
+	/// The notifications substream has been accepted by the remote.
+	Open {
+		/// Handshake message sent by the remote after we opened the substream.
+		handshake: Vec<u8>,
+	},
+
+	/// The notifications substream has been closed by the remote.
+	Closed,
+
+	/// We tried to open a notifications substream, but the remote refused it.
+	///
+	/// Can only happen if we're in a closed state.
+	Refused,
+}
+
+impl NotifsOutHandler {
+	/// Returns true if the substream is currently open.
+	pub fn is_open(&self) -> bool {
+		match &self.state {
+			State::Disabled => false,
+			State::DisabledOpening => false,
+			State::DisabledOpen(_) => true,
+			State::Opening { .. } => false,
+			State::Refused => false,
+			State::Open { .. } => true,
+			State::Poisoned => false,
+		}
+	}
+
+	/// Returns the name of the protocol that we negotiate.
+	pub fn protocol_name(&self) -> &[u8] {
+		&self.protocol_name
+	}
+}
+
+impl ProtocolsHandler for NotifsOutHandler {
+	type InEvent = NotifsOutHandlerIn;
+	type OutEvent = NotifsOutHandlerOut;
+	type Error = void::Void;
+	type InboundProtocol = DeniedUpgrade;
+	type OutboundProtocol = NotificationsOut;
+	type OutboundOpenInfo = ();
+
+	fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+		SubstreamProtocol::new(DeniedUpgrade)
+	}
+
+	fn inject_fully_negotiated_inbound(
+		&mut self,
+		proto: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output
+	) {
+		// We should never reach here. `proto` is a `Void`.
+		void::unreachable(proto)
+	}
+
+	fn inject_fully_negotiated_outbound(
+		&mut self,
+		(handshake_msg, substream): <Self::OutboundProtocol as OutboundUpgrade<NegotiatedSubstream>>::Output,
+		_: ()
+	) {
+		match mem::replace(&mut self.state, State::Poisoned) {
+			State::Opening { initial_message } => {
+				let ev = NotifsOutHandlerOut::Open { handshake: handshake_msg };
+				self.events_queue.push(ProtocolsHandlerEvent::Custom(ev));
+				self.state = State::Open { substream, initial_message };
+			},
+			// If the handler was disabled while we were negotiating the protocol, immediately
+			// close it.
+			State::DisabledOpening => self.state = State::DisabledOpen(substream),
+
+			// Any other situation should never happen.
+			State::Disabled | State::Refused | State::Open { .. } | State::DisabledOpen(_) =>
+				error!("State mismatch in notifications handler: substream already open"),
+			State::Poisoned => error!("Notifications handler in a poisoned state"),
+		}
+	}
+
+	fn inject_event(&mut self, message: NotifsOutHandlerIn) {
+		match message {
+			NotifsOutHandlerIn::Enable { initial_message } => {
+				match mem::replace(&mut self.state, State::Poisoned) {
+					State::Disabled => {
+						let proto = NotificationsOut::new(self.protocol_name.clone(), initial_message.clone());
+						self.events_queue.push(ProtocolsHandlerEvent::OutboundSubstreamRequest {
+							protocol: SubstreamProtocol::new(proto).with_timeout(OPEN_TIMEOUT),
+							info: (),
+						});
+						self.state = State::Opening { initial_message };
+					},
+					State::DisabledOpening => self.state = State::Opening { initial_message },
+					State::DisabledOpen(mut sub) => {
+						// As documented above, in this state we have already called `poll_close`
+						// once on the substream, and it is unclear whether the substream can then
+						// be recovered. When in doubt, let's drop the existing substream and
+						// open a new one.
+						if sub.close().now_or_never().is_none() {
+							log::warn!(
+								target: "sub-libp2p",
+								"Improperly closed outbound notifications substream"
+							);
+						}
+
+						let proto = NotificationsOut::new(self.protocol_name.clone(), initial_message.clone());
+						self.events_queue.push(ProtocolsHandlerEvent::OutboundSubstreamRequest {
+							protocol: SubstreamProtocol::new(proto).with_timeout(OPEN_TIMEOUT),
+							info: (),
+						});
+						self.state = State::Opening { initial_message };
+					},
+					State::Opening { .. } | State::Refused | State::Open { .. } =>
+						error!("Tried to enable notifications handler that was already enabled"),
+					State::Poisoned => error!("Notifications handler in a poisoned state"),
+				}
+			}
+
+			NotifsOutHandlerIn::Disable => {
+				match mem::replace(&mut self.state, State::Poisoned) {
+					State::Disabled | State::DisabledOpen(_) | State::DisabledOpening =>
+						error!("Tried to disable notifications handler that was already disabled"),
+					State::Opening { .. } => self.state = State::DisabledOpening,
+					State::Refused => self.state = State::Disabled,
+					State::Open { substream, .. } => self.state = State::DisabledOpen(substream),
+					State::Poisoned => error!("Notifications handler in a poisoned state"),
+				}
+			}
+
+			NotifsOutHandlerIn::Send(msg) =>
+				if let State::Open { substream, .. } = &mut self.state {
+					if let Some(Ok(_)) = substream.send(msg).now_or_never() {
+					} else {
+						log::warn!(
+							target: "sub-libp2p",
+							"Failed to push message to queue, dropped it"
+						);
+					}
+				} else {
+					// This is an API misuse.
+					log::warn!(
+						target: "sub-libp2p",
+						"Tried to send a notification on a disabled handler"
+					);
+				},
+		}
+	}
+
+	fn inject_dial_upgrade_error(&mut self, _: (), _: ProtocolsHandlerUpgrErr<NotificationsHandshakeError>) {
+		match mem::replace(&mut self.state, State::Poisoned) {
+			State::Disabled => {},
+			State::DisabledOpen(_) | State::Refused | State::Open { .. } =>
+				error!("State mismatch in NotificationsOut"),
+			State::Opening { .. } => {
+				self.state = State::Refused;
+				let ev = NotifsOutHandlerOut::Refused;
+				self.events_queue.push(ProtocolsHandlerEvent::Custom(ev));
+			},
+			State::DisabledOpening => self.state = State::Disabled,
+			State::Poisoned => error!("Notifications handler in a poisoned state"),
+		}
+	}
+
+	fn connection_keep_alive(&self) -> KeepAlive {
+		match self.state {
+			// We have a small grace period of `INITIAL_KEEPALIVE_TIME` during which we keep the
+			// connection open no matter what, in order to avoid closing and reopening
+			// connections all the time.
+			State::Disabled | State::DisabledOpen(_) | State::DisabledOpening =>
+				KeepAlive::Until(self.when_connection_open + INITIAL_KEEPALIVE_TIME),
+			State::Opening { .. } | State::Open { .. } => KeepAlive::Yes,
+			State::Refused | State::Poisoned => KeepAlive::No,
+		}
+	}
+
+	fn poll(
+		&mut self,
+		cx: &mut Context,
+	) -> Poll<ProtocolsHandlerEvent<Self::OutboundProtocol, Self::OutboundOpenInfo, Self::OutEvent, Self::Error>> {
+		// Flush the events queue if necessary.
+		if !self.events_queue.is_empty() {
+			let event = self.events_queue.remove(0);
+			return Poll::Ready(event);
+		}
+
+		match &mut self.state {
+			State::Open { substream, initial_message } =>
+				match Sink::poll_flush(Pin::new(substream), cx) {
+					Poll::Pending | Poll::Ready(Ok(())) => {},
+					Poll::Ready(Err(_)) => {
+						// We try to re-open a substream.
+						let initial_message = mem::replace(initial_message, Vec::new());
+						self.state = State::Opening { initial_message: initial_message.clone() };
+						let proto = NotificationsOut::new(self.protocol_name.clone(), initial_message);
+						self.events_queue.push(ProtocolsHandlerEvent::OutboundSubstreamRequest {
+							protocol: SubstreamProtocol::new(proto).with_timeout(OPEN_TIMEOUT),
+							info: (),
+						});
+						return Poll::Ready(ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Closed));
+					}
+				},
+
+			State::DisabledOpen(sub) => match Sink::poll_close(Pin::new(sub), cx) {
+				Poll::Pending => {},
+				Poll::Ready(Ok(())) | Poll::Ready(Err(_)) => {
+					self.state = State::Disabled;
+					return Poll::Ready(ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Closed));
+				},
+			},
+
+			_ => {}
+		}
+
+		Poll::Pending
+	}
+}
+
+impl fmt::Debug for NotifsOutHandler {
+	fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+		f.debug_struct("NotifsOutHandler")
+			.field("open", &self.is_open())
+			.finish()
+	}
+}