Add a back-pressure-friendly alternative to NetworkService::write_notifications 🎉 (#6692)

* Add NetworkService::send_notifications

* Doc

* Doc

* API adjustment

* Address concerns

* Make it compile

* Start implementation

* Progress in the implementation

* Change implementation strategy again

* More work before weekend

* Finish changes

* Minor doc fix

* Revert some minor changes

* Apply suggestions from code review

* GroupError -> NotifsHandlerError

* Apply suggestions from code review

Co-authored-by: Roman Borschel <romanb@users.noreply.github.com>

* state_transition_waker -> close_waker

* Apply suggestions from code review

Co-authored-by: Roman Borschel <romanb@users.noreply.github.com>

* Finish renames in service.rs

* More renames

* More review suggestsions applied

* More review addressing

* Final change

* 512 -> 2048

Co-authored-by: Roman Borschel <romanb@users.noreply.github.com>

substrate/client/network/src/service/tests.rs

@@ -345,6 +345,57 @@ fn lots_of_incoming_peers_works() {
 	});
 }
 
+#[test]
+fn notifications_back_pressure() {
+	// Node 1 floods node 2 with notifications. Random sleeps are done on node 2 to simulate the
+	// node being busy. We make sure that all notifications are received.
+
+	const TOTAL_NOTIFS: usize = 10_000;
+
+	let (node1, mut events_stream1, node2, mut events_stream2) = build_nodes_one_proto();
+	let node2_id = node2.local_peer_id();
+
+	let receiver = async_std::task::spawn(async move {
+		let mut received_notifications = 0;
+
+		while received_notifications < TOTAL_NOTIFS {
+			match events_stream2.next().await.unwrap() {
+				Event::NotificationStreamClosed { .. } => panic!(),
+				Event::NotificationsReceived { messages, .. } => {
+					for message in messages {
+						assert_eq!(message.0, ENGINE_ID);
+						assert_eq!(message.1, format!("hello #{}", received_notifications));
+						received_notifications += 1;
+					}
+				}
+				_ => {}
+			};
+
+			if rand::random::<u8>() < 2 {
+				async_std::task::sleep(Duration::from_millis(rand::random::<u64>() % 750)).await;
+			}
+		}
+	});
+
+	async_std::task::block_on(async move {
+		// Wait for the `NotificationStreamOpened`.
+		loop {
+			match events_stream1.next().await.unwrap() {
+				Event::NotificationStreamOpened { .. } => break,
+				_ => {}
+			};
+		}
+
+		// Sending!
+		for num in 0..TOTAL_NOTIFS {
+			let notif = node1.notification_sender(node2_id.clone(), ENGINE_ID).unwrap();
+			notif.ready().await.unwrap().send(format!("hello #{}", num)).unwrap();
+		}
+
+		receiver.await;
+	});
+}
+
 #[test]
 #[should_panic(expected = "don't match the transport")]
 fn ensure_listen_addresses_consistent_with_transport_memory() {