Add more tests, fix (and test for) a deadlock re overquota messages, more unbounded channels and less .awaiting

backend/test_utils/src/feed_message_de.rs

@@ -1,3 +1,4 @@
+use anyhow::Context;
 use common::node_types::{
     BlockDetails, BlockHash, BlockNumber, NodeLocation, NodeStats, Timestamp,
 };
@@ -22,7 +23,7 @@ pub enum FeedMessage {
         // io: NodeIO, // can't losslessly deserialize
         // hardware: NodeHardware, // can't losslessly deserialize
         block_details: BlockDetails,
-        location: NodeLocation,
+        location: Option<NodeLocation>,
         startup_time: Option<Timestamp>,
     },
     RemovedNode {
@@ -127,15 +128,18 @@ impl FeedMessage {
         for raw_keyval in v.chunks(2) {
             let raw_key = raw_keyval[0];
             let raw_val = raw_keyval[1];
-            feed_messages.push(FeedMessage::decode(raw_key, raw_val)?);
+            let action: u8 = serde_json::from_str(raw_key.get())?;
+            let msg = FeedMessage::decode(action, raw_val)
+                .with_context(|| format!("Failed to decode message with action {}", action))?;
+
+            feed_messages.push(msg);
         }
 
         Ok(feed_messages)
     }
 
     // Deserialize the feed message to a value based on the "action" key
-    fn decode(raw_key: &RawValue, raw_val: &RawValue) -> Result<FeedMessage, anyhow::Error> {
-        let action: u8 = serde_json::from_str(raw_key.get())?;
+    fn decode(action: u8, raw_val: &RawValue) -> Result<FeedMessage, anyhow::Error> {
         let feed_message = match action {
             // Version:
             0 => {

backend/test_utils/src/server/utils.rs

@@ -65,15 +65,15 @@ pub async fn connect_multiple_to_uri(
     uri: &http::Uri,
     num_connections: usize,
 ) -> Result<Vec<(ws_client::Sender, ws_client::Receiver)>, ws_client::ConnectError> {
-
-    // Batch connection establishing to groups of 100 at a time; I found while benchmarking that
-    // I'd run into "connection reset by peer" issues trying to establish more at once.
-    let connect_futs = (0..num_connections).map(|_| ws_client::connect(uri));
-    let sockets: Result<Vec<_>, _> = futures::future::join_all(connect_futs)
-        .await
-        .into_iter()
-        .collect();
-    sockets
+    // Previous versions of this used future::join_all to concurrently establish all of the
+    // connections we want. However, trying to do that with more than say ~130 connections on
+    // MacOS led to hitting "Connection reset by peer" errors, so let's do it one-at-a-time.
+    // (Side note: on Ubuntu the concurrency seemed to be no issue up to at least 10k connections).
+    let mut sockets = vec![];
+    for _ in 0..num_connections {
+        sockets.push(ws_client::connect(uri).await?);
+    }
+    Ok(sockets)
 }
 
 /// Drain output from a reader to stdout. After acquiring port details from spawned processes,