Avoid using flume::Receiver::into_stream() to avoid memory leaks until the issue is resolved upstream (#394)

* Tweak rolling_total test to also confirm capacity doesn't go nuts

* Use Jemalloc

* Avoid flume's into_stream and use a workaround for now

* cargo fmt

* Improve comments now that there's an issue to point to

backend/common/src/flume_recv_stream.rs

@@ -0,0 +1,21 @@
+//! A sort-of drop-in replacement to create a Stream from a flume Receiver, because `flume::Receiver::into_stream()`
+//! leaks memory. See:
+//!
+//! https://github.com/zesterer/flume/issues/88
+//!
+//! Hopefully we won't need to use these for long; the issue will probably be resolved fairly prompty and we can
+//! revert back to using the built-in flume methods.
+//!
+use flume::Receiver;
+use futures::stream::poll_fn;
+use futures::{FutureExt, Stream};
+use std::pin::Pin;
+
+/// A drop-in replacement which is similar to `flume::RecvStream`.
+pub type FlumeRecvStream<'a, T> = Pin<Box<dyn Stream<Item = T> + Send + 'a>>;
+
+/// A drop-in replacement for `flume`'s `Receiver::into_stream()` method.
+pub fn flume_receiver_into_stream<'a, T: Send + 'a>(r: Receiver<T>) -> FlumeRecvStream<'a, T> {
+    let stream = poll_fn(move |cx| r.recv_async().poll_unpin(cx).map(|r| r.ok()));
+    Box::pin(stream)
+}

backend/common/src/lib.rs

@@ -28,6 +28,7 @@ pub mod ws_client;
 mod assign_id;
 mod dense_map;
 mod either_sink;
+mod flume_recv_stream;
 mod mean_list;
 mod most_seen;
 mod multi_map_unique;
@@ -37,6 +38,7 @@ mod num_stats;
 pub use assign_id::AssignId;
 pub use dense_map::DenseMap;
 pub use either_sink::EitherSink;
+pub use flume_recv_stream::{flume_receiver_into_stream, FlumeRecvStream};
 pub use mean_list::MeanList;
 pub use most_seen::MostSeen;
 pub use multi_map_unique::MultiMapUnique;

backend/common/src/rolling_total.rs

@@ -215,6 +215,7 @@ mod test {
         }
 
         assert_eq!(rolling_total.averages().len(), 3);
+        assert!(rolling_total.averages().capacity() < 10); // Just to show that it's capacity is bounded.
     }
 
     #[test]

backend/common/src/ws_client/connect.rs

@@ -205,7 +205,7 @@ impl Connection {
                 closer: Arc::clone(&on_close),
             },
             Receiver {
-                inner: rx_from_ws.into_stream(),
+                inner: crate::flume_receiver_into_stream(rx_from_ws),
                 closer: on_close,
             },
         )

backend/common/src/ws_client/receiver.rs

@@ -20,7 +20,7 @@ use std::sync::Arc;
 
 /// Receive messages out of a connection
 pub struct Receiver {
-    pub(super) inner: flume::r#async::RecvStream<'static, Result<RecvMessage, RecvError>>,
+    pub(super) inner: crate::FlumeRecvStream<'static, Result<RecvMessage, RecvError>>,
     pub(super) closer: Arc<OnClose>,
 }