Avoid using flume::Receiver::into_stream() to avoid memory leaks until the issue is resolved upstream (#394)

* Tweak rolling_total test to also confirm capacity doesn't go nuts

* Use Jemalloc

* Avoid flume's into_stream and use a workaround for now

* cargo fmt

* Improve comments now that there's an issue to point to
This commit is contained in:
James Wilson
2021-09-03 08:40:43 +01:00
committed by GitHub
parent a4069e4b3d
commit 2932075783
10 changed files with 80 additions and 7 deletions
+12 -5
View File
@@ -34,6 +34,13 @@ use hyper::{Method, Response};
use simple_logger::SimpleLogger;
use structopt::StructOpt;
#[cfg(not(target_env = "msvc"))]
use jemallocator::Jemalloc;
#[cfg(not(target_env = "msvc"))]
#[global_allocator]
static GLOBAL: Jemalloc = Jemalloc;
const VERSION: &str = env!("CARGO_PKG_VERSION");
const AUTHORS: &str = env!("CARGO_PKG_AUTHORS");
const NAME: &str = "Substrate Telemetry Backend Core";
@@ -204,7 +211,6 @@ where
S: futures::Sink<FromShardWebsocket, Error = anyhow::Error> + Unpin + Send + 'static,
{
let (tx_to_shard_conn, rx_from_aggregator) = flume::unbounded();
let mut rx_from_aggregator = rx_from_aggregator.into_stream();
// Tell the aggregator about this new connection, and give it a way to send messages to us:
let init_msg = FromShardWebsocket::Initialize {
@@ -291,13 +297,13 @@ where
let send_handle = tokio::spawn(async move {
loop {
let msg = tokio::select! {
msg = rx_from_aggregator.next() => msg,
msg = rx_from_aggregator.recv_async() => msg,
_ = &mut send_closer_rx => { break }
};
let msg = match msg {
Some(msg) => msg,
None => break,
Ok(msg) => msg,
Err(flume::RecvError::Disconnected) => break,
};
let internal_msg = match msg {
@@ -347,7 +353,8 @@ where
{
// unbounded channel so that slow feeds don't block aggregator progress:
let (tx_to_feed_conn, rx_from_aggregator) = flume::unbounded();
let mut rx_from_aggregator_chunks = ReadyChunksAll::new(rx_from_aggregator.into_stream());
let mut rx_from_aggregator_chunks =
ReadyChunksAll::new(common::flume_receiver_into_stream(rx_from_aggregator));
// Tell the aggregator about this new connection, and give it a way to send messages to us:
let init_msg = FromFeedWebsocket::Initialize {