*: Bump async-std to v1.6.5 (#7306)

* *: Bump async-std to v1.6.5

Prevent users from using v1.6.4 which faces issues receiving incoming
TCP connections. See https://github.com/async-rs/async-std/issues/888
for details.

* client/network/src/gossip: Use channel instead of condvar

`async_std::sync::Condvar::wait_timeout` uses
`gloo_timers::callback::Timeout` when compiled for
`wasm32-unknown-unknown`. This timeout implementation does not fulfill
the requirement of being `Send`.

Instead of using a `Condvar` use a `futures::channel::mpsc` to signal
progress from the `QueuedSender` to the background `Future`.

* client/network/Cargo.toml: Remove async-std unstable feature

* client/network/src/gossip: Forward all queued messages

* client/network/gossip: Have QueuedSender methods take &mut self

* client/network/gossip: Move queue_size_limit into QueuedSender

The `queue_size_limit` field is only accessed by `QueuedSender`, thus
there is no need to share it between the background future and the
`QueuedSender`.

* client/network/gossip: Rename background task to future

To be a bit picky the background task is not a task in the sense of an
asynchonous task, but rather a background future in the sense of
`futures::future::Future`.
This commit is contained in:
Max Inden
2020-10-20 11:23:27 +02:00
committed by GitHub
parent bd2dbc055c
commit 918a0c8077
7 changed files with 191 additions and 146 deletions
+117 -55
View File
@@ -220,9 +220,9 @@ checksum = "7deb0a829ca7bcfaf5da70b073a8d128619259a7be8216a355e23f00763059e5"
[[package]]
name = "async-channel"
version = "1.1.1"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee81ba99bee79f3c8ae114ae4baa7eaa326f63447cf2ec65e4393618b63f8770"
checksum = "59740d83946db6a5af71ae25ddf9562c2b176b2ca42cf99a455f09f4a220d6b9"
dependencies = [
"concurrent-queue",
"event-listener",
@@ -230,34 +230,95 @@ dependencies = [
]
[[package]]
name = "async-std"
version = "1.6.2"
name = "async-executor"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00d68a33ebc8b57800847d00787307f84a562224a14db069b0acefe4c2abbf5d"
checksum = "d373d78ded7d0b3fa8039375718cde0aace493f2e34fb60f51cbf567562ca801"
dependencies = [
"async-task",
"concurrent-queue",
"fastrand",
"futures-lite",
"once_cell 1.4.1",
"vec-arena",
]
[[package]]
name = "async-global-executor"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fefeb39da249f4c33af940b779a56723ce45809ef5c54dad84bb538d4ffb6d9e"
dependencies = [
"async-executor",
"async-io",
"futures-lite",
"num_cpus",
"once_cell 1.4.1",
]
[[package]]
name = "async-io"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38628c78a34f111c5a6b98fc87dfc056cd1590b61afe748b145be4623c56d194"
dependencies = [
"cfg-if",
"concurrent-queue",
"fastrand",
"futures-lite",
"libc",
"log",
"once_cell 1.4.1",
"parking",
"polling",
"socket2",
"vec-arena",
"waker-fn",
"wepoll-sys-stjepang",
"winapi 0.3.9",
]
[[package]]
name = "async-mutex"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479db852db25d9dbf6204e6cb6253698f175c15726470f78af0d918e99d6156e"
dependencies = [
"event-listener",
]
[[package]]
name = "async-std"
version = "1.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9fa76751505e8df1c7a77762f60486f60c71bbd9b8557f4da6ad47d083732ed"
dependencies = [
"async-global-executor",
"async-io",
"async-mutex",
"blocking",
"crossbeam-utils",
"futures-channel",
"futures-core",
"futures-io",
"futures-timer 3.0.2",
"futures-lite",
"gloo-timers",
"kv-log-macro",
"log",
"memchr",
"num_cpus",
"once_cell 1.4.0",
"once_cell 1.4.1",
"pin-project-lite",
"pin-utils",
"slab",
"smol",
"wasm-bindgen-futures",
]
[[package]]
name = "async-task"
version = "3.0.0"
version = "4.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c17772156ef2829aadc587461c7753af20b7e8db1529bc66855add962a3b35d3"
checksum = "8ab27c1aa62945039e44edaeee1dc23c74cc0c303dd5fe0fb462a184f1c3a518"
[[package]]
name = "async-tls"
@@ -506,16 +567,16 @@ dependencies = [
[[package]]
name = "blocking"
version = "0.4.7"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2468ff7bf85066b4a3678fede6fe66db31846d753ff0adfbfab2c6a6e81612b"
checksum = "c5e170dbede1f740736619b776d7251cb1b9095c435c34d8ca9f57fcd2f335e9"
dependencies = [
"async-channel",
"async-task",
"atomic-waker",
"fastrand",
"futures-lite",
"once_cell 1.4.0",
"parking",
"waker-fn",
"once_cell 1.4.1",
]
[[package]]
@@ -745,9 +806,9 @@ dependencies = [
[[package]]
name = "concurrent-queue"
version = "1.1.1"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83c06aff61f2d899eb87c379df3cbf7876f14471dcab474e0b6dc90ab96c080"
checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3"
dependencies = [
"cache-padded",
]
@@ -1333,9 +1394,9 @@ dependencies = [
[[package]]
name = "event-listener"
version = "2.2.0"
version = "2.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "699d84875f1b72b4da017e6b0f77dfa88c0137f089958a88974d15938cbc2976"
checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59"
[[package]]
name = "evm"
@@ -1428,9 +1489,12 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
[[package]]
name = "fastrand"
version = "1.3.3"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36a9cb09840f81cd211e435d00a4e487edd263dc3c8ff815c32dd76ad668ebed"
checksum = "ca5faf057445ce5c9d4329e382b2ce7ca38550ef3b73a5348362d5f24e0c7fe3"
dependencies = [
"instant",
]
[[package]]
name = "fdlimit"
@@ -1588,7 +1652,7 @@ dependencies = [
"frame-system",
"impl-trait-for-tuples",
"log",
"once_cell 1.4.0",
"once_cell 1.4.1",
"parity-scale-codec",
"parity-util-mem",
"paste 0.1.18",
@@ -1843,9 +1907,9 @@ checksum = "de27142b013a8e869c14957e6d2edeef89e97c289e69d042ee3a49acd8b51789"
[[package]]
name = "futures-lite"
version = "0.1.8"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "180d8fc9819eb48a0c976672fbeea13a73e10999e812bdc9e14644c25ad51d60"
checksum = "381a7ad57b1bad34693f63f6f377e1abded7a9c85c9d3eb6771e11c60aaadab9"
dependencies = [
"fastrand",
"futures-core",
@@ -1880,7 +1944,7 @@ version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bdb66b5f09e22019b1ab0830f7785bcea8e7a42148683f99214f73f8ec21a626"
dependencies = [
"once_cell 1.4.0",
"once_cell 1.4.1",
]
[[package]]
@@ -2750,9 +2814,9 @@ checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a"
[[package]]
name = "libc"
version = "0.2.73"
version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd7d4bd64732af4bf3a67f367c27df8520ad7e230c5817b8ff485864d80242b9"
checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743"
[[package]]
name = "libloading"
@@ -4155,11 +4219,11 @@ dependencies = [
[[package]]
name = "once_cell"
version = "1.4.0"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d"
checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad"
dependencies = [
"parking_lot 0.10.2",
"parking_lot 0.11.0",
]
[[package]]
@@ -5198,9 +5262,9 @@ dependencies = [
[[package]]
name = "parking"
version = "1.0.5"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50d4a6da31f8144a32532fe38fe8fb439a6842e0ec633f0037f0144c14e7f907"
checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72"
[[package]]
name = "parking_lot"
@@ -5427,6 +5491,19 @@ dependencies = [
"web-sys",
]
[[package]]
name = "polling"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0720e0b9ea9d52451cf29d3413ba8a9303f8815d9d9653ef70e03ff73e65566"
dependencies = [
"cfg-if",
"libc",
"log",
"wepoll-sys-stjepang",
"winapi 0.3.9",
]
[[package]]
name = "poly1305"
version = "0.6.0"
@@ -6035,7 +6112,7 @@ checksum = "952cd6b98c85bbc30efa1ba5783b8abf12fec8b3287ffa52605b9432313e34e4"
dependencies = [
"cc",
"libc",
"once_cell 1.4.0",
"once_cell 1.4.1",
"spin",
"untrusted",
"web-sys",
@@ -7729,27 +7806,6 @@ version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3757cb9d89161a2f24e1cf78efa0c1fcff485d18e3f55e0aa3480824ddaa0f3f"
[[package]]
name = "smol"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "620cbb3c6e34da57d3a248cda0cd01cd5848164dc062e764e65d06fe3ea7aed5"
dependencies = [
"async-task",
"blocking",
"concurrent-queue",
"fastrand",
"futures-io",
"futures-util",
"libc",
"once_cell 1.4.0",
"scoped-tls",
"slab",
"socket2",
"wepoll-sys-stjepang",
"winapi 0.3.9",
]
[[package]]
name = "snow"
version = "0.7.1"
@@ -9078,7 +9134,7 @@ checksum = "b0165e045cc2ae1660270ca65e1676dbaab60feb0f91b10f7d0665e9b47e31f2"
dependencies = [
"failure",
"hmac",
"once_cell 1.4.0",
"once_cell 1.4.1",
"pbkdf2",
"rand 0.7.3",
"rustc-hash",
@@ -9718,6 +9774,12 @@ version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c"
[[package]]
name = "vec-arena"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eafc1b9b2dfc6f5529177b62cf806484db55b32dc7c9658a118e11bbeb33061d"
[[package]]
name = "vec_map"
version = "0.8.2"
+1 -1
View File
@@ -25,7 +25,7 @@ sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" }
wasm-timer = "0.2"
[dev-dependencies]
async-std = "1.6.2"
async-std = "1.6.5"
quickcheck = "0.9.0"
rand = "0.7.2"
substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" }
+1 -1
View File
@@ -18,7 +18,7 @@ prost-build = "0.6.1"
[dependencies]
async-trait = "0.1"
async-std = { version = "1.6.2", features = ["unstable"] }
async-std = "1.6.5"
bitflags = "1.2.0"
bs58 = "0.3.1"
bytes = "0.5.0"
+69 -86
View File
@@ -49,15 +49,15 @@
use crate::{ExHashT, NetworkService};
use async_std::sync::{Condvar, Mutex, MutexGuard};
use async_std::sync::{Mutex, MutexGuard};
use futures::prelude::*;
use futures::channel::mpsc::{channel, Receiver, Sender};
use libp2p::PeerId;
use sp_runtime::{traits::Block as BlockT, ConsensusEngineId};
use std::{
collections::VecDeque,
fmt,
sync::{atomic, Arc},
time::Duration,
sync::Arc,
};
#[cfg(test)]
@@ -65,8 +65,12 @@ mod tests;
/// Notifications sender for a specific combination of network service, peer, and protocol.
pub struct QueuedSender<M> {
/// Shared between the front and the back task.
shared: Arc<Shared<M>>,
/// Shared between the user-facing [`QueuedSender`] and the background future.
shared_message_queue: SharedMessageQueue<M>,
/// Used to notify the background future to check for new messages in the message queue.
notify_background_future: Sender<()>,
/// Maximum number of elements in [`QueuedSender::shared_message_queue`].
queue_size_limit: usize,
}
impl<M> QueuedSender<M> {
@@ -88,39 +92,45 @@ impl<M> QueuedSender<M> {
H: ExHashT,
F: Fn(M) -> Vec<u8> + Send + 'static,
{
let shared = Arc::new(Shared {
stop_task: atomic::AtomicBool::new(false),
condvar: Condvar::new(),
queue_size_limit,
messages_queue: Mutex::new(VecDeque::with_capacity(queue_size_limit)),
});
let (notify_background_future, wait_for_sender) = channel(0);
let task = spawn_task(
let shared_message_queue = Arc::new(Mutex::new(
VecDeque::with_capacity(queue_size_limit),
));
let background_future = create_background_future(
wait_for_sender,
service,
peer_id,
protocol,
shared.clone(),
shared_message_queue.clone(),
messages_encode
);
(QueuedSender { shared }, task)
let sender = QueuedSender {
shared_message_queue,
notify_background_future,
queue_size_limit,
};
(sender, background_future)
}
/// Locks the queue of messages towards this peer.
///
/// The returned `Future` is expected to be ready quite quickly.
pub async fn lock_queue<'a>(&'a self) -> QueueGuard<'a, M> {
pub async fn lock_queue<'a>(&'a mut self) -> QueueGuard<'a, M> {
QueueGuard {
messages_queue: self.shared.messages_queue.lock().await,
condvar: &self.shared.condvar,
queue_size_limit: self.shared.queue_size_limit,
message_queue: self.shared_message_queue.lock().await,
queue_size_limit: self.queue_size_limit,
notify_background_future: &mut self.notify_background_future,
}
}
/// Pushes a message to the queue, or discards it if the queue is full.
///
/// The returned `Future` is expected to be ready quite quickly.
pub async fn queue_or_discard(&self, message: M)
pub async fn queue_or_discard(&mut self, message: M)
where
M: Send + 'static
{
@@ -134,28 +144,17 @@ impl<M> fmt::Debug for QueuedSender<M> {
}
}
impl<M> Drop for QueuedSender<M> {
fn drop(&mut self) {
// The "clean" way to notify the `Condvar` here is normally to first lock the `Mutex`,
// then notify the `Condvar` while the `Mutex` is locked. Unfortunately, the `Mutex`
// being asynchronous, it can't reasonably be locked from within a destructor.
// See also the corresponding code in the background task.
self.shared.stop_task.store(true, atomic::Ordering::Release);
self.shared.condvar.notify_all();
}
}
/// Locked queue of messages to the given peer.
///
/// As long as this struct exists, the background task is asleep and the owner of the [`QueueGuard`]
/// is in total control of the buffer. Messages can only ever be sent out after the [`QueueGuard`]
/// is dropped.
/// As long as this struct exists, the background future is asleep and the owner of the
/// [`QueueGuard`] is in total control of the message queue. Messages can only ever be sent out on
/// the network after the [`QueueGuard`] is dropped.
#[must_use]
pub struct QueueGuard<'a, M> {
messages_queue: MutexGuard<'a, VecDeque<M>>,
condvar: &'a Condvar,
/// Same as [`Shared::queue_size_limit`].
message_queue: MutexGuard<'a, MessageQueue<M>>,
/// Same as [`QueuedSender::queue_size_limit`].
queue_size_limit: usize,
notify_background_future: &'a mut Sender<()>,
}
impl<'a, M: Send + 'static> QueueGuard<'a, M> {
@@ -163,8 +162,8 @@ impl<'a, M: Send + 'static> QueueGuard<'a, M> {
///
/// The message will only start being sent out after the [`QueueGuard`] is dropped.
pub fn push_or_discard(&mut self, message: M) {
if self.messages_queue.len() < self.queue_size_limit {
self.messages_queue.push_back(message);
if self.message_queue.len() < self.queue_size_limit {
self.message_queue.push_back(message);
}
}
@@ -174,72 +173,56 @@ impl<'a, M: Send + 'static> QueueGuard<'a, M> {
/// > **Note**: The parameter of `filter` is a `&M` and not a `&mut M` (which would be
/// > better) because the underlying implementation relies on `VecDeque::retain`.
pub fn retain(&mut self, filter: impl FnMut(&M) -> bool) {
self.messages_queue.retain(filter);
self.message_queue.retain(filter);
}
}
impl<'a, M> Drop for QueueGuard<'a, M> {
fn drop(&mut self) {
// We notify the `Condvar` in the destructor in order to be able to push multiple
// messages and wake up the background task only once afterwards.
self.condvar.notify_one();
// Notify background future to check for new messages in the message queue.
let _ = self.notify_background_future.try_send(());
}
}
#[derive(Debug)]
struct Shared<M> {
/// Read by the background task after locking `locked`. If true, the task stops.
stop_task: atomic::AtomicBool,
/// Queue of messages waiting to be sent out.
messages_queue: Mutex<VecDeque<M>>,
/// Must be notified every time the content of `locked` changes.
condvar: Condvar,
/// Maximum number of elements in `messages_queue`.
queue_size_limit: usize,
}
type MessageQueue<M> = VecDeque<M>;
async fn spawn_task<B: BlockT, H: ExHashT, M, F: Fn(M) -> Vec<u8>>(
/// [`MessageQueue`] shared between [`QueuedSender`] and background future.
type SharedMessageQueue<M> = Arc<Mutex<MessageQueue<M>>>;
async fn create_background_future<B: BlockT, H: ExHashT, M, F: Fn(M) -> Vec<u8>>(
mut wait_for_sender: Receiver<()>,
service: Arc<NetworkService<B, H>>,
peer_id: PeerId,
protocol: ConsensusEngineId,
shared: Arc<Shared<M>>,
shared_message_queue: SharedMessageQueue<M>,
messages_encode: F,
) {
loop {
let next_message = 'next_msg: loop {
let mut queue = shared.messages_queue.lock().await;
if wait_for_sender.next().await.is_none() {
return
}
loop {
if shared.stop_task.load(atomic::Ordering::Acquire) {
return;
}
loop {
let mut queue_guard = shared_message_queue.lock().await;
let next_message = match queue_guard.pop_front() {
Some(msg) => msg,
None => break,
};
drop(queue_guard);
if let Some(msg) = queue.pop_front() {
break 'next_msg msg;
}
// Starting from below, we try to send the message. If an error happens when sending,
// the only sane option we have is to silently discard the message.
let sender = match service.notification_sender(peer_id.clone(), protocol) {
Ok(s) => s,
Err(_) => continue,
};
// It is possible that the destructor of `QueuedSender` sets `stop_task` to
// true and notifies the `Condvar` after the background task loads `stop_task`
// and before it calls `Condvar::wait`.
// See also the corresponding comment in `QueuedSender::drop`.
// For this reason, we use `wait_timeout`. In the worst case scenario,
// `stop_task` will always be checked again after the timeout is reached.
queue = shared.condvar.wait_timeout(queue, Duration::from_secs(10)).await.0;
}
};
let ready = match sender.ready().await {
Ok(r) => r,
Err(_) => continue,
};
// Starting from below, we try to send the message. If an error happens when sending,
// the only sane option we have is to silently discard the message.
let sender = match service.notification_sender(peer_id.clone(), protocol) {
Ok(s) => s,
Err(_) => continue,
};
let ready = match sender.ready().await {
Ok(r) => r,
Err(_) => continue,
};
let _ = ready.send(messages_encode(next_message));
let _ = ready.send(messages_encode(next_message));
}
}
}
+1 -1
View File
@@ -180,7 +180,7 @@ fn basic_works() {
});
async_std::task::block_on(async move {
let (sender, bg_future) =
let (mut sender, bg_future) =
QueuedSender::new(node1, node2_id, ENGINE_ID, NUM_NOTIFS, |msg| msg);
async_std::task::spawn(bg_future);
+1 -1
View File
@@ -89,4 +89,4 @@ sp-consensus-babe = { version = "0.8.0", path = "../../primitives/consensus/babe
grandpa = { version = "0.8.0", package = "sc-finality-grandpa", path = "../finality-grandpa" }
grandpa-primitives = { version = "2.0.0", package = "sp-finality-grandpa", path = "../../primitives/finality-grandpa" }
tokio = { version = "0.2", default-features = false }
async-std = { version = "1.6", default-features = false }
async-std = { version = "1.6.5", default-features = false }
+1 -1
View File
@@ -19,6 +19,6 @@ futures-util = { version = "0.3.1", default-features = false, features = ["io"]
derive_more = "0.99"
[target.'cfg(not(target_os = "unknown"))'.dependencies]
async-std = { version = "1.6.2", features = ["unstable"] }
async-std = { version = "1.6.5", features = ["unstable"] }
hyper = { version = "0.13.1", default-features = false, features = ["stream"] }
tokio = "0.2"