pezkuwichain/pezkuwi-telemetry
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-telemetry.git synced 2026-06-13 01:01:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

cargo fmt

Browse Source
This commit is contained in:
James Wilson
2021-07-27 12:38:05 +01:00
parent a44f39cfaf
commit 2ca5f6a1fb
23 changed files with 707 additions and 511 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/telemetry_core/benches/throughput.rs
+80 -80
View File
@@ -1,98 +1,98 @@
use std::iter::FromIterator;
use futures::StreamExt;
use test_utils::workspace::start_server_release;
use criterion::{criterion_group, criterion_main, Criterion};
use tokio::runtime::Runtime;
use serde_json::json;
use common::node_types::BlockHash;
use criterion::{criterion_group, criterion_main, Criterion};
use futures::StreamExt;
use serde_json::json;
use test_utils::workspace::start_server_release;
use tokio::runtime::Runtime;
pub fn benchmark_throughput_single_shard(c: &mut Criterion) {
/*
let rt = Runtime::new().expect("tokio runtime should start");
let rt = Runtime::new().expect("tokio runtime should start");
// Setup our server and node/feed connections first:
let (nodes, feeds) = rt.block_on(async {
let mut server = start_server_release().await;
let shard_id = server.add_shard().await.unwrap();
// Setup our server and node/feed connections first:
let (nodes, feeds) = rt.block_on(async {
let mut server = start_server_release().await;
let shard_id = server.add_shard().await.unwrap();
// Connect 1000 nodes to the shard:
let mut nodes = server
.get_shard(shard_id)
.unwrap()
.connect_multiple(1000)
.await
.expect("nodes can connect");
// Connect 1000 nodes to the shard:
let mut nodes = server
.get_shard(shard_id)
.unwrap()
.connect_multiple(1000)
.await
.expect("nodes can connect");
// Every node announces itself on the same chain:
for (idx, (node_tx, _)) in nodes.iter_mut().enumerate() {
node_tx.send_json_text(json!({
"id":1, // message ID, not node ID. Can be the same for all.
"ts":"2021-07-12T10:37:47.714666+01:00",
"payload": {
"authority":true,
"chain":"Local Testnet",
"config":"",
"genesis_hash": BlockHash::from_low_u64_ne(1),
"implementation":"Substrate Node",
"msg":"system.connected",
"name": format!("Alice {}", idx),
"network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
"startup_time":"1625565542717",
"version":"2.0.0-07a1af348-aarch64-macos"
}
})).await.unwrap();
// Every node announces itself on the same chain:
for (idx, (node_tx, _)) in nodes.iter_mut().enumerate() {
node_tx.send_json_text(json!({
"id":1, // message ID, not node ID. Can be the same for all.
"ts":"2021-07-12T10:37:47.714666+01:00",
"payload": {
"authority":true,
"chain":"Local Testnet",
"config":"",
"genesis_hash": BlockHash::from_low_u64_ne(1),
"implementation":"Substrate Node",
"msg":"system.connected",
"name": format!("Alice {}", idx),
"network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
"startup_time":"1625565542717",
"version":"2.0.0-07a1af348-aarch64-macos"
}
})).await.unwrap();
}
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
// Start 1000 feeds:
let mut feeds = server
.get_core()
.connect_multiple(1)
.await
.expect("feeds can connect");
// // Subscribe all feeds to the chain:
// for (feed_tx, _) in feeds.iter_mut() {
// feed_tx.send_command("subscribe", "Local Testnet").await.unwrap();
// }
println!("consuming feed");
{
let mut msgs = futures::stream::FuturesUnordered::from_iter(
feeds
.iter_mut()
.map(|(_,rx)| rx.recv_feed_messages())
);
let mut n = 0;
while let Some(Ok(msg)) = msgs.next().await {
n += 1;
println!("Message {}: {:?}", n, msg);
}
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
// Start 1000 feeds:
let mut feeds = server
.get_core()
.connect_multiple(1)
.await
.expect("feeds can connect");
// // Subscribe all feeds to the chain:
// for (feed_tx, _) in feeds.iter_mut() {
// feed_tx.send_command("subscribe", "Local Testnet").await.unwrap();
// }
println!("consuming feed");
{
let mut msgs = futures::stream::FuturesUnordered::from_iter(
feeds
.iter_mut()
.map(|(_,rx)| rx.recv_feed_messages())
);
let mut n = 0;
while let Some(Ok(msg)) = msgs.next().await {
n += 1;
println!("Message {}: {:?}", n, msg);
}
}
// // Consume any messages feeds have received so far (every feed should havea few at least):
// let feed_consumers = feeds
// .iter_mut()
// .map(|(_,rx)| rx.next());
// futures::future::join_all(feed_consumers).await;
println!("feed consumed");
(nodes, feeds)
});
// // Consume any messages feeds have received so far (every feed should havea few at least):
// let feed_consumers = feeds
// .iter_mut()
// .map(|(_,rx)| rx.next());
// futures::future::join_all(feed_consumers).await;
println!("feed consumed");
(nodes, feeds)
});
// Next, run criterion using the same tokio runtime to benchmark time taken to send
// messages to nodes and receive them from feeds.
c.bench_function(
"throughput time",
|b| b.to_async(&rt).iter(|| async {
// Next, run criterion using the same tokio runtime to benchmark time taken to send
// messages to nodes and receive them from feeds.
c.bench_function(
"throughput time",
|b| b.to_async(&rt).iter(|| async {
// TODO: Actually implement the benchmark.
// TODO: Actually implement the benchmark.
})
);
*/
})
);
*/
}
criterion_group!(benches, benchmark_throughput_single_shard);
criterion_main!(benches);
criterion_main!(benches);
backend/telemetry_core/src/aggregator/aggregator.rs
+4 -2
View File
@@ -74,7 +74,8 @@ impl Aggregator {
/// Return a sink that a shard can send messages into to be handled by the aggregator.
pub fn subscribe_shard(
&self,
) -> impl Sink<inner_loop::FromShardWebsocket, Error = anyhow::Error> + Send + Sync + Unpin + 'static {
) -> impl Sink<inner_loop::FromShardWebsocket, Error = anyhow::Error> + Send + Sync + Unpin + 'static
{
// Assign a unique aggregator-local ID to each connection that subscribes, and pass
// that along with every message to the aggregator loop:
let shard_conn_id = self
@@ -96,7 +97,8 @@ impl Aggregator {
/// Return a sink that a feed can send messages into to be handled by the aggregator.
pub fn subscribe_feed(
&self,
) -> impl Sink<inner_loop::FromFeedWebsocket, Error = anyhow::Error> + Send + Sync + Unpin + 'static {
) -> impl Sink<inner_loop::FromFeedWebsocket, Error = anyhow::Error> + Send + Sync + Unpin + 'static
{
// Assign a unique aggregator-local ID to each connection that subscribes, and pass
// that along with every message to the aggregator loop:
let feed_conn_id = self
backend/telemetry_core/src/aggregator/inner_loop.rs
+10 -19
View File
@@ -179,11 +179,7 @@ impl InnerLoop {
}
/// Handle messages that come from the node geographical locator.
fn handle_from_find_location(
&mut self,
node_id: NodeId,
location: find_location::Location,
) {
fn handle_from_find_location(&mut self, node_id: NodeId, location: find_location::Location) {
self.node_state
.update_node_location(node_id, location.clone());
@@ -227,20 +223,18 @@ impl InnerLoop {
match self.node_state.add_node(genesis_hash, node) {
state::AddNodeResult::ChainOnDenyList => {
if let Some(shard_conn) = self.shard_channels.get_mut(&shard_conn_id) {
let _ = shard_conn
.unbounded_send(ToShardWebsocket::Mute {
local_id,
reason: MuteReason::ChainNotAllowed,
});
let _ = shard_conn.unbounded_send(ToShardWebsocket::Mute {
local_id,
reason: MuteReason::ChainNotAllowed,
});
}
}
state::AddNodeResult::ChainOverQuota => {
if let Some(shard_conn) = self.shard_channels.get_mut(&shard_conn_id) {
let _ = shard_conn
.unbounded_send(ToShardWebsocket::Mute {
local_id,
reason: MuteReason::Overquota,
});
let _ = shard_conn.unbounded_send(ToShardWebsocket::Mute {
local_id,
reason: MuteReason::Overquota,
});
}
}
state::AddNodeResult::NodeAddedToChain(details) => {
@@ -473,10 +467,7 @@ impl InnerLoop {
}
/// Remove all of the node IDs provided and broadcast messages to feeds as needed.
fn remove_nodes_and_broadcast_result(
&mut self,
node_ids: impl IntoIterator<Item = NodeId>,
) {
fn remove_nodes_and_broadcast_result(&mut self, node_ids: impl IntoIterator<Item = NodeId>) {
// Group by chain to simplify the handling of feed messages:
let mut node_ids_per_chain: HashMap<BlockHash, Vec<NodeId>> = HashMap::new();
for node_id in node_ids.into_iter() {
backend/telemetry_core/src/main.rs
+99 -61
View File
@@ -3,19 +3,19 @@ mod feed_message;
mod find_location;
mod state;
use std::str::FromStr;
use tokio::time::{ Duration, Instant };
use tokio::time::{Duration, Instant};
use aggregator::{
Aggregator, FromFeedWebsocket, FromShardWebsocket, ToFeedWebsocket, ToShardWebsocket,
};
use bincode::Options;
use common::http_utils;
use common::internal_messages;
use common::ready_chunks_all::ReadyChunksAll;
use futures::{channel::mpsc, SinkExt, StreamExt};
use hyper::{Method, Response};
use simple_logger::SimpleLogger;
use structopt::StructOpt;
use hyper::{ Response, Method };
use common::http_utils;
const VERSION: &str = env!("CARGO_PKG_VERSION");
const AUTHORS: &str = env!("CARGO_PKG_AUTHORS");
@@ -42,7 +42,7 @@ struct Opts {
/// If it takes longer than this number of seconds to send the current batch of messages
/// to a feed, the feed connection will be closed.
#[structopt(long, default_value = "10")]
feed_timeout: u64
feed_timeout: u64,
}
#[tokio::main]
@@ -72,40 +72,55 @@ async fn start_server(opts: Opts) -> anyhow::Result<()> {
async move {
match (req.method(), req.uri().path().trim_end_matches('/')) {
// Check that the server is up and running:
(&Method::GET, "/health") => {
Ok(Response::new("OK".into()))
},
(&Method::GET, "/health") => Ok(Response::new("OK".into())),
// Subscribe to feed messages:
(&Method::GET, "/feed") => {
Ok(http_utils::upgrade_to_websocket(req, move |ws_send, ws_recv| async move {
let tx_to_aggregator = aggregator.subscribe_feed();
let (mut tx_to_aggregator, mut ws_send)
= handle_feed_websocket_connection(ws_send, ws_recv, tx_to_aggregator, feed_timeout).await;
log::info!("Closing /feed connection from {:?}", addr);
// Tell the aggregator that this connection has closed, so it can tidy up.
let _ = tx_to_aggregator.send(FromFeedWebsocket::Disconnected).await;
let _ = ws_send.close().await;
}))
},
Ok(http_utils::upgrade_to_websocket(
req,
move |ws_send, ws_recv| async move {
let tx_to_aggregator = aggregator.subscribe_feed();
let (mut tx_to_aggregator, mut ws_send) =
handle_feed_websocket_connection(
ws_send,
ws_recv,
tx_to_aggregator,
feed_timeout,
)
.await;
log::info!("Closing /feed connection from {:?}", addr);
// Tell the aggregator that this connection has closed, so it can tidy up.
let _ = tx_to_aggregator.send(FromFeedWebsocket::Disconnected).await;
let _ = ws_send.close().await;
},
))
}
// Subscribe to shard messages:
(&Method::GET, "/shard_submit") => {
Ok(http_utils::upgrade_to_websocket(req, move |ws_send, ws_recv| async move {
let tx_to_aggregator = aggregator.subscribe_shard();
let (mut tx_to_aggregator, mut ws_send)
= handle_shard_websocket_connection(ws_send, ws_recv, tx_to_aggregator).await;
log::info!("Closing /shard_submit connection from {:?}", addr);
// Tell the aggregator that this connection has closed, so it can tidy up.
let _ = tx_to_aggregator.send(FromShardWebsocket::Disconnected).await;
let _ = ws_send.close().await;
}))
},
// 404 for anything else:
_ => {
Ok(Response::builder()
.status(404)
.body("Not found".into())
.unwrap())
Ok(http_utils::upgrade_to_websocket(
req,
move |ws_send, ws_recv| async move {
let tx_to_aggregator = aggregator.subscribe_shard();
let (mut tx_to_aggregator, mut ws_send) =
handle_shard_websocket_connection(
ws_send,
ws_recv,
tx_to_aggregator,
)
.await;
log::info!("Closing /shard_submit connection from {:?}", addr);
// Tell the aggregator that this connection has closed, so it can tidy up.
let _ = tx_to_aggregator
.send(FromShardWebsocket::Disconnected)
.await;
let _ = ws_send.close().await;
},
))
}
// 404 for anything else:
_ => Ok(Response::builder()
.status(404)
.body("Not found".into())
.unwrap()),
}
}
});
@@ -156,29 +171,44 @@ where
break;
}
if let Err(e) = msg_info {
log::error!("Shutting down websocket connection: Failed to receive data: {}", e);
log::error!(
"Shutting down websocket connection: Failed to receive data: {}",
e
);
break;
}
let msg: internal_messages::FromShardAggregator = match bincode::options().deserialize(&bytes) {
Ok(msg) => msg,
Err(e) => {
log::error!("Failed to deserialize message from shard; booting it: {}", e);
break;
}
};
let msg: internal_messages::FromShardAggregator =
match bincode::options().deserialize(&bytes) {
Ok(msg) => msg,
Err(e) => {
log::error!(
"Failed to deserialize message from shard; booting it: {}",
e
);
break;
}
};
// Convert and send to the aggregator:
let aggregator_msg = match msg {
internal_messages::FromShardAggregator::AddNode { ip, node, local_id, genesis_hash } => {
FromShardWebsocket::Add { ip, node, genesis_hash, local_id }
internal_messages::FromShardAggregator::AddNode {
ip,
node,
local_id,
genesis_hash,
} => FromShardWebsocket::Add {
ip,
node,
genesis_hash,
local_id,
},
internal_messages::FromShardAggregator::UpdateNode { payload, local_id } => {
FromShardWebsocket::Update { local_id, payload }
},
}
internal_messages::FromShardAggregator::RemoveNode { local_id } => {
FromShardWebsocket::Remove { local_id }
},
}
};
if let Err(e) = tx_to_aggregator.send(aggregator_msg).await {
@@ -201,7 +231,7 @@ where
let msg = match msg {
Some(msg) => msg,
None => break
None => break,
};
let internal_msg = match msg {
@@ -218,9 +248,11 @@ where
log::error!("Failed to send message to aggregator; closing shard: {}", e)
}
if let Err(e) = ws_send.flush().await {
log::error!("Failed to flush message to aggregator; closing shard: {}", e)
log::error!(
"Failed to flush message to aggregator; closing shard: {}",
e
)
}
}
drop(recv_closer_tx); // Kill the recv task if this send task ends
@@ -241,7 +273,7 @@ async fn handle_feed_websocket_connection<S>(
mut ws_send: http_utils::WsSender,
mut ws_recv: http_utils::WsReceiver,
mut tx_to_aggregator: S,
feed_timeout: u64
feed_timeout: u64,
) -> (S, http_utils::WsSender)
where
S: futures::Sink<FromFeedWebsocket, Error = anyhow::Error> + Unpin + Send + 'static,
@@ -281,29 +313,35 @@ where
break;
}
if let Err(e) = msg_info {
log::error!("Shutting down websocket connection: Failed to receive data: {}", e);
log::error!(
"Shutting down websocket connection: Failed to receive data: {}",
e
);
break;
}
// We ignore all but valid UTF8 text messages from the frontend:
let text = match String::from_utf8(bytes) {
Ok(s) => s,
Err(_) => continue
Err(_) => continue,
};
// Parse the message into a command we understand and send it to the aggregator:
let cmd = match FromFeedWebsocket::from_str(&text) {
Ok(cmd) => cmd,
Err(e) => {
log::warn!("Ignoring invalid command '{}' from the frontend: {}", text, e);
continue
log::warn!(
"Ignoring invalid command '{}' from the frontend: {}",
text,
e
);
continue;
}
};
if let Err(e) = tx_to_aggregator.send(cmd).await {
log::error!("Failed to send message to aggregator; closing feed: {}", e);
break;
}
}
drop(send_closer_tx); // Kill the send task if this recv task ends
@@ -323,15 +361,13 @@ where
// End the loop when connection from aggregator ends:
let msgs = match msgs {
Some(msgs) => msgs,
None => break
None => break,
};
// There is only one message type at the mo; bytes to send
// to the websocket. collect them all up to dispatch in one shot.
let all_msg_bytes = msgs.into_iter().map(|msg| {
match msg {
ToFeedWebsocket::Bytes(bytes) => bytes
}
let all_msg_bytes = msgs.into_iter().map(|msg| match msg {
ToFeedWebsocket::Bytes(bytes) => bytes,
});
// We have a deadline to send and flush messages. If the client isn't keeping up with our
@@ -340,7 +376,9 @@ where
let message_send_deadline = Instant::now() + Duration::from_secs(feed_timeout);
for bytes in all_msg_bytes {
match tokio::time::timeout_at(message_send_deadline, ws_send.send_binary(&bytes)).await {
match tokio::time::timeout_at(message_send_deadline, ws_send.send_binary(&bytes))
.await
{
Err(_) => {
log::warn!("Closing feed websocket that was too slow to keep up (1)");
break 'outer;
@@ -355,7 +393,7 @@ where
match tokio::time::timeout_at(message_send_deadline, ws_send.flush()).await {
Err(_) => {
log::warn!("Closing feed websocket that was too slow to keep up (2)");
break
break;
}
Ok(Err(e)) => {
log::warn!("Closing feed websocket due to error flushing data: {}", e);
backend/telemetry_core/tests/e2e_tests.rs
+199 -107
View File
@@ -6,7 +6,7 @@ use std::time::Duration;
use test_utils::{
assert_contains_matches,
feed_message_de::{FeedMessage, NodeDetails},
workspace::{ start_server, CoreOpts, ShardOpts, start_server_debug }
workspace::{start_server, start_server_debug, CoreOpts, ShardOpts},
};
/// The simplest test we can run; the main benefit of this test (since we check similar)
@@ -55,7 +55,6 @@ async fn feed_ping_responded_to_with_pong() {
server.shutdown().await;
}
/// As a prelude to `lots_of_mute_messages_dont_cause_a_deadlock`, we can check that
/// a lot of nodes can simultaneously subscribe and are all sent the expected response.
#[tokio::test]
@@ -70,10 +69,8 @@ async fn multiple_feeds_sent_version_on_connect() {
.unwrap();
// Wait for responses all at once:
let responses = futures::future::join_all(
feeds.iter_mut()
.map(|(_, rx)| rx.recv_feed_messages())
);
let responses =
futures::future::join_all(feeds.iter_mut().map(|(_, rx)| rx.recv_feed_messages()));
let responses = tokio::time::timeout(Duration::from_secs(10), responses)
.await
@@ -113,22 +110,24 @@ async fn lots_of_mute_messages_dont_cause_a_deadlock() {
// Every node announces itself on the same chain:
for (idx, (node_tx, _)) in nodes.iter_mut().enumerate() {
node_tx.send_json_text(json!({
"id":1, // message ID, not node ID. Can be the same for all.
"ts":"2021-07-12T10:37:47.714666+01:00",
"payload": {
"authority":true,
"chain":"Local Testnet",
"config":"",
"genesis_hash": BlockHash::from_low_u64_ne(1),
"implementation":"Substrate Node",
"msg":"system.connected",
"name": format!("Alice {}", idx),
"network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
"startup_time":"1625565542717",
"version":"2.0.0-07a1af348-aarch64-macos"
}
})).unwrap();
node_tx
.send_json_text(json!({
"id":1, // message ID, not node ID. Can be the same for all.
"ts":"2021-07-12T10:37:47.714666+01:00",
"payload": {
"authority":true,
"chain":"Local Testnet",
"config":"",
"genesis_hash": BlockHash::from_low_u64_ne(1),
"implementation":"Substrate Node",
"msg":"system.connected",
"name": format!("Alice {}", idx),
"network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
"startup_time":"1625565542717",
"version":"2.0.0-07a1af348-aarch64-macos"
}
}))
.unwrap();
}
// Wait a little time (just to let everything get deadlocked) before
@@ -144,11 +143,8 @@ async fn lots_of_mute_messages_dont_cause_a_deadlock() {
.expect("feeds can connect");
// Wait to see whether we get anything back:
let msgs_fut = futures::future::join_all(
feeds
.iter_mut()
.map(|(_,rx)| rx.recv_feed_messages())
);
let msgs_fut =
futures::future::join_all(feeds.iter_mut().map(|(_, rx)| rx.recv_feed_messages()));
// Give up after a timeout:
tokio::time::timeout(Duration::from_secs(10), msgs_fut)
@@ -234,29 +230,35 @@ async fn feeds_told_about_chain_rename_and_stay_subscribed() {
.await
.expect("can connect to shard");
let node_init_msg = |id, chain_name: &str, node_name: &str| json!({
"id":id,
"ts":"2021-07-12T10:37:47.714666+01:00",
"payload": {
"authority":true,
"chain": chain_name,
"config":"",
"genesis_hash": BlockHash::from_low_u64_ne(1),
"implementation":"Substrate Node",
"msg":"system.connected",
"name": node_name,
"network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
"startup_time":"1625565542717",
"version":"2.0.0-07a1af348-aarch64-macos"
},
});
let node_init_msg = |id, chain_name: &str, node_name: &str| {
json!({
"id":id,
"ts":"2021-07-12T10:37:47.714666+01:00",
"payload": {
"authority":true,
"chain": chain_name,
"config":"",
"genesis_hash": BlockHash::from_low_u64_ne(1),
"implementation":"Substrate Node",
"msg":"system.connected",
"name": node_name,
"network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
"startup_time":"1625565542717",
"version":"2.0.0-07a1af348-aarch64-macos"
},
})
};
// Subscribe a chain:
node_tx.send_json_text(node_init_msg(1, "Initial chain name", "Node 1")).unwrap();
node_tx
.send_json_text(node_init_msg(1, "Initial chain name", "Node 1"))
.unwrap();
// Connect a feed and subscribe to the above chain:
let (mut feed_tx, mut feed_rx) = server.get_core().connect_feed().await.unwrap();
feed_tx.send_command("subscribe", "Initial chain name").unwrap();
feed_tx
.send_command("subscribe", "Initial chain name")
.unwrap();
// Feed is told about the chain, and the node on this chain:
let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
@@ -269,7 +271,9 @@ async fn feeds_told_about_chain_rename_and_stay_subscribed() {
// Subscribe another node. The chain doesn't rename yet but we are told about the new node
// count and the node that's been added.
node_tx.send_json_text(node_init_msg(2, "New chain name", "Node 2")).unwrap();
node_tx
.send_json_text(node_init_msg(2, "New chain name", "Node 2"))
.unwrap();
let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
assert_contains_matches!(
feed_messages,
@@ -279,7 +283,9 @@ async fn feeds_told_about_chain_rename_and_stay_subscribed() {
// Subscribe a third node. The chain renames, so we're told about the new node but also
// about the chain rename.
node_tx.send_json_text(node_init_msg(3, "New chain name", "Node 3")).unwrap();
node_tx
.send_json_text(node_init_msg(3, "New chain name", "Node 3"))
.unwrap();
let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
assert_contains_matches!(
feed_messages,
@@ -290,14 +296,15 @@ async fn feeds_told_about_chain_rename_and_stay_subscribed() {
// Just to be sure, subscribing a fourth node on this chain will still lead to updates
// to this feed.
node_tx.send_json_text(node_init_msg(4, "New chain name", "Node 4")).unwrap();
node_tx
.send_json_text(node_init_msg(4, "New chain name", "Node 4"))
.unwrap();
let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
assert_contains_matches!(
feed_messages,
FeedMessage::AddedNode { node: NodeDetails { name: node_name, .. }, ..} if node_name == "Node 4",
FeedMessage::AddedChain { name, node_count: 4 } if name == "New chain name",
);
}
/// If we add a couple of shards and a node for each, all feeds should be
@@ -386,7 +393,12 @@ async fn feed_can_subscribe_and_unsubscribe_from_chain() {
// Start server, add shard, connect node:
let mut server = start_server_debug().await;
let shard_id = server.add_shard().await.unwrap();
let (mut node_tx, _node_rx) = server.get_shard(shard_id).unwrap().connect_node().await.unwrap();
let (mut node_tx, _node_rx) = server
.get_shard(shard_id)
.unwrap()
.connect_node()
.await
.unwrap();
// Send a "system connected" message for a few nodes/chains:
for id in 1..=3 {
@@ -419,7 +431,9 @@ async fn feed_can_subscribe_and_unsubscribe_from_chain() {
assert_contains_matches!(feed_messages, AddedChain { name, node_count: 1 } if name == "Local Testnet 1");
// Subscribe it to a chain
feed_tx.send_command("subscribe", "Local Testnet 1").unwrap();
feed_tx
.send_command("subscribe", "Local Testnet 1")
.unwrap();
let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
assert_contains_matches!(
@@ -450,7 +464,9 @@ async fn feed_can_subscribe_and_unsubscribe_from_chain() {
.expect_err("Timeout should elapse since no messages sent");
// We can change our subscription:
feed_tx.send_command("subscribe", "Local Testnet 2").unwrap();
feed_tx
.send_command("subscribe", "Local Testnet 2")
.unwrap();
let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
// We are told about the subscription change and given similar on-subscribe messages to above.
@@ -484,36 +500,48 @@ async fn slow_feeds_are_disconnected() {
let mut server = start_server(
true,
// Timeout faster so the test can be quicker:
CoreOpts { feed_timeout: Some(1) },
CoreOpts {
feed_timeout: Some(1),
},
// Allow us to send more messages in more easily:
ShardOpts { max_nodes_per_connection: Some(100_000) }
).await;
ShardOpts {
max_nodes_per_connection: Some(100_000),
},
)
.await;
// Give us a shard to talk to:
let shard_id = server.add_shard().await.unwrap();
let (mut node_tx, _node_rx) = server.get_shard(shard_id).unwrap().connect_node().await.unwrap();
let (mut node_tx, _node_rx) = server
.get_shard(shard_id)
.unwrap()
.connect_node()
.await
.unwrap();
// Add a load of nodes from this shard so there's plenty of data to give to a feed.
// We want to exhaust any buffers between core and feed (eg BufWriters). If the number
// is too low, data will happily be sent into a buffer and the connection won't need to
// be closed.
for n in 1..50_000 {
node_tx.send_json_text(json!({
"id":n,
"ts":"2021-07-12T10:37:47.714666+01:00",
"payload": {
"authority":true,
"chain":"Polkadot",
"config":"",
"genesis_hash": BlockHash::from_low_u64_ne(1),
"implementation":"Substrate Node",
"msg":"system.connected",
"name": format!("Alice {}", n),
"network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
"startup_time":"1625565542717",
"version":"2.0.0-07a1af348-aarch64-macos"
}
})).unwrap();
node_tx
.send_json_text(json!({
"id":n,
"ts":"2021-07-12T10:37:47.714666+01:00",
"payload": {
"authority":true,
"chain":"Polkadot",
"config":"",
"genesis_hash": BlockHash::from_low_u64_ne(1),
"implementation":"Substrate Node",
"msg":"system.connected",
"name": format!("Alice {}", n),
"network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
"startup_time":"1625565542717",
"version":"2.0.0-07a1af348-aarch64-macos"
}
}))
.unwrap();
}
// Connect a raw feed so that we can control how fast we consume data from the websocket
@@ -530,10 +558,8 @@ async fn slow_feeds_are_disconnected() {
// waiting to receive mroe data (or see some other error).
loop {
let mut v = Vec::new();
let data = tokio::time::timeout(
Duration::from_secs(1),
raw_feed_rx.receive_data(&mut v)
).await;
let data =
tokio::time::timeout(Duration::from_secs(1), raw_feed_rx.receive_data(&mut v)).await;
match data {
Ok(Ok(_)) => {
@@ -541,13 +567,13 @@ async fn slow_feeds_are_disconnected() {
}
Ok(Err(soketto::connection::Error::Closed)) => {
break; // End loop; success!
},
}
Ok(Err(e)) => {
panic!("recv should be closed but instead we saw this error: {}", e);
},
}
Err(_) => {
panic!("recv should be closed but seems to be happy waiting for more data");
},
}
}
}
@@ -564,49 +590,87 @@ async fn max_nodes_per_connection_is_enforced() {
false,
CoreOpts::default(),
// Limit max nodes per connection to 2; any other msgs should be ignored.
ShardOpts { max_nodes_per_connection: Some(2) }
).await;
ShardOpts {
max_nodes_per_connection: Some(2),
},
)
.await;
// Connect to a shard
let shard_id = server.add_shard().await.unwrap();
let (mut node_tx, _node_rx) = server.get_shard(shard_id).unwrap().connect_node().await.unwrap();
let (mut node_tx, _node_rx) = server
.get_shard(shard_id)
.unwrap()
.connect_node()
.await
.unwrap();
// Connect a feed.
let (mut feed_tx, mut feed_rx) = server.get_core().connect_feed().await.unwrap();
// We'll send these messages from the node:
let json_msg = |n| json!({
"id":n,
"ts":"2021-07-12T10:37:47.714666+01:00",
"payload": {
"authority":true,
"chain":"Test Chain",
"config":"",
"genesis_hash": BlockHash::from_low_u64_ne(1),
"implementation":"Polkadot",
"msg":"system.connected",
"name": format!("Alice {}", n),
"network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
"startup_time":"1625565542717",
"version":"2.0.0-07a1af348-aarch64-macos"
}
});
let json_msg = |n| {
json!({
"id":n,
"ts":"2021-07-12T10:37:47.714666+01:00",
"payload": {
"authority":true,
"chain":"Test Chain",
"config":"",
"genesis_hash": BlockHash::from_low_u64_ne(1),
"implementation":"Polkadot",
"msg":"system.connected",
"name": format!("Alice {}", n),
"network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
"startup_time":"1625565542717",
"version":"2.0.0-07a1af348-aarch64-macos"
}
})
};
// First message ID should lead to feed messages:
node_tx.send_json_text(json_msg(1)).unwrap();
assert_ne!(feed_rx.recv_feed_messages_timeout(Duration::from_secs(1)).await.unwrap().len(), 0);
assert_ne!(
feed_rx
.recv_feed_messages_timeout(Duration::from_secs(1))
.await
.unwrap()
.len(),
0
);
// Second message ID should lead to feed messages as well:
node_tx.send_json_text(json_msg(2)).unwrap();
assert_ne!(feed_rx.recv_feed_messages_timeout(Duration::from_secs(1)).await.unwrap().len(), 0);
assert_ne!(
feed_rx
.recv_feed_messages_timeout(Duration::from_secs(1))
.await
.unwrap()
.len(),
0
);
// Third message ID should be ignored:
node_tx.send_json_text(json_msg(3)).unwrap();
assert_eq!(feed_rx.recv_feed_messages_timeout(Duration::from_secs(1)).await.unwrap().len(), 0);
assert_eq!(
feed_rx
.recv_feed_messages_timeout(Duration::from_secs(1))
.await
.unwrap()
.len(),
0
);
// Forth message ID should be ignored as well:
node_tx.send_json_text(json_msg(4)).unwrap();
assert_eq!(feed_rx.recv_feed_messages_timeout(Duration::from_secs(1)).await.unwrap().len(), 0);
assert_eq!(
feed_rx
.recv_feed_messages_timeout(Duration::from_secs(1))
.await
.unwrap()
.len(),
0
);
// (now that the chain "Test Chain" is known about, subscribe to it for update messages.
// This wasn't needed to receive messages re the above since everybody hears about node
@@ -619,25 +683,53 @@ async fn max_nodes_per_connection_is_enforced() {
node_tx.send_json_text(json!(
{"id":1, "payload":{ "bandwidth_download":576,"bandwidth_upload":576,"msg":"system.interval","peers":1},"ts":"2021-07-12T10:38:48.330433+01:00" }
)).unwrap();
assert_ne!(feed_rx.recv_feed_messages_timeout(Duration::from_secs(1)).await.unwrap().len(), 0);
assert_ne!(
feed_rx
.recv_feed_messages_timeout(Duration::from_secs(1))
.await
.unwrap()
.len(),
0
);
node_tx.send_json_text(json!(
{"id":2, "payload":{ "bandwidth_download":576,"bandwidth_upload":576,"msg":"system.interval","peers":1},"ts":"2021-07-12T10:38:48.330433+01:00" }
)).unwrap();
assert_ne!(feed_rx.recv_feed_messages_timeout(Duration::from_secs(1)).await.unwrap().len(), 0);
assert_ne!(
feed_rx
.recv_feed_messages_timeout(Duration::from_secs(1))
.await
.unwrap()
.len(),
0
);
// Updates about ignored IDs are still ignored:
node_tx.send_json_text(json!(
{"id":3, "payload":{ "bandwidth_download":576,"bandwidth_upload":576,"msg":"system.interval","peers":1},"ts":"2021-07-12T10:38:48.330433+01:00" }
)).unwrap();
assert_eq!(feed_rx.recv_feed_messages_timeout(Duration::from_secs(1)).await.unwrap().len(), 0);
assert_eq!(
feed_rx
.recv_feed_messages_timeout(Duration::from_secs(1))
.await
.unwrap()
.len(),
0
);
node_tx.send_json_text(json!(
{"id":4, "payload":{ "bandwidth_download":576,"bandwidth_upload":576,"msg":"system.interval","peers":1},"ts":"2021-07-12T10:38:48.330433+01:00" }
)).unwrap();
assert_eq!(feed_rx.recv_feed_messages_timeout(Duration::from_secs(1)).await.unwrap().len(), 0);
assert_eq!(
feed_rx
.recv_feed_messages_timeout(Duration::from_secs(1))
.await
.unwrap()
.len(),
0
);
// Tidy up:
server.shutdown().await;
}
}
backend/telemetry_core/tests/soak_tests.rs
+37 -32
View File
@@ -18,15 +18,15 @@ In general, if you run into issues, it may be better to run this on a linux
box; MacOS seems to hit limits quicker in general.
*/
use futures::{ StreamExt };
use common::node_types::BlockHash;
use common::ws_client::SentMessage;
use futures::StreamExt;
use serde_json::json;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
use structopt::StructOpt;
use test_utils::workspace::start_server_release;
use common::ws_client::{ SentMessage };
use serde_json::json;
use std::time::Duration;
use std::sync::atomic::{ Ordering, AtomicUsize };
use std::sync::Arc;
use common::node_types::BlockHash;
/// A configurable soak_test runner. Configure by providing the expected args as
/// an environment variable. One example to run this test is:
@@ -78,22 +78,24 @@ async fn run_soak_test(opts: SoakTestOpts) {
// Each node tells the shard about itself:
for (idx, (node_tx, _)) in nodes.iter_mut().enumerate() {
node_tx.send_json_binary(json!({
"id":1, // Only needs to be unique per node
"ts":"2021-07-12T10:37:47.714666+01:00",
"payload": {
"authority":true,
"chain": "Polkadot", // <- so that we don't go over quota with lots of nodes.
"config":"",
"genesis_hash": BlockHash::from_low_u64_ne(1),
"implementation":"Substrate Node",
"msg":"system.connected",
"name": format!("Node #{}", idx),
"network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
"startup_time":"1625565542717",
"version":"2.0.0-07a1af348-aarch64-macos"
},
})).unwrap();
node_tx
.send_json_binary(json!({
"id":1, // Only needs to be unique per node
"ts":"2021-07-12T10:37:47.714666+01:00",
"payload": {
"authority":true,
"chain": "Polkadot", // <- so that we don't go over quota with lots of nodes.
"config":"",
"genesis_hash": BlockHash::from_low_u64_ne(1),
"implementation":"Substrate Node",
"msg":"system.connected",
"name": format!("Node #{}", idx),
"network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
"startup_time":"1625565542717",
"version":"2.0.0-07a1af348-aarch64-macos"
},
}))
.unwrap();
}
// Connect feeds to the core:
@@ -127,12 +129,15 @@ async fn run_soak_test(opts: SoakTestOpts) {
loop {
// every ~1second we aim to have sent messages from all of the nodes. So we cycle through
// the node IDs and send a message from each at roughly 1s / number_of_nodes.
let mut interval = tokio::time::interval(Duration::from_secs_f64(1.0 / nodes.len() as f64));
let mut interval =
tokio::time::interval(Duration::from_secs_f64(1.0 / nodes.len() as f64));
for node_id in (0..nodes.len()).cycle() {
interval.tick().await;
let node_tx = &mut nodes[node_id].0;
node_tx.unbounded_send(SentMessage::StaticBinary(msg_bytes)).unwrap();
node_tx
.unbounded_send(SentMessage::StaticBinary(msg_bytes))
.unwrap();
bytes_in2.fetch_add(msg_bytes.len(), Ordering::Relaxed);
}
}
@@ -162,7 +167,8 @@ async fn run_soak_test(opts: SoakTestOpts) {
let bytes_in_val = bytes_in.load(Ordering::Relaxed);
let bytes_out_val = bytes_out.load(Ordering::Relaxed);
println!("#{}: MB in/out per measurement: {:.4} / {:.4}, total bytes in/out: {} / {})",
println!(
"#{}: MB in/out per measurement: {:.4} / {:.4}, total bytes in/out: {} / {})",
n,
(bytes_in_val - last_bytes_in) as f64 / one_mb,
(bytes_out_val - last_bytes_out) as f64 / one_mb,
@@ -193,19 +199,18 @@ struct SoakTestOpts {
feeds: usize,
/// The number of nodes to connect to each feed
#[structopt(long)]
nodes: usize
nodes: usize,
}
/// Get soak test args from an envvar and parse them via structopt.
fn get_soak_test_opts() -> SoakTestOpts {
let arg_string = std::env::var("SOAK_TEST_ARGS")
.expect("Expecting args to be provided in the env var SOAK_TEST_ARGS");
let args = shellwords::split(&arg_string)
.expect("Could not parse SOAK_TEST_ARGS as shell arguments");
let args =
shellwords::split(&arg_string).expect("Could not parse SOAK_TEST_ARGS as shell arguments");
// The binary name is expected to be the first arg, so fake it:
let all_args = std::iter::once("soak_test".to_owned())
.chain(args.into_iter());
let all_args = std::iter::once("soak_test".to_owned()).chain(args.into_iter());
SoakTestOpts::from_iter(all_args)
}
}