pezkuwichain/pezkuwi-telemetry
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-telemetry.git synced 2026-06-14 03:41:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

cargo fmt

Browse Source
This commit is contained in:
James Wilson
2021-08-06 17:44:26 +01:00
parent 74cf55174e
commit 88c3db3562
17 changed files with 134 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/telemetry_core/benches/subscribe.rs
+30 -28
View File
@@ -1,10 +1,10 @@
use common::node_types::BlockHash;
use criterion::{criterion_group, criterion_main, Criterion};
use serde_json::json;
use test_utils::workspace::{ start_server, ServerOpts, CoreOpts, ShardOpts };
use test_utils::feed_message_de::FeedMessage;
use tokio::runtime::Runtime;
use std::time::{Duration, Instant};
use test_utils::feed_message_de::FeedMessage;
use test_utils::workspace::{start_server, CoreOpts, ServerOpts, ShardOpts};
use tokio::runtime::Runtime;
/// This benchmark roughly times the subscribe function. Note that there's a lot of
/// overhead in other areas, so even with the entire subscribe function commented out
@@ -19,10 +19,8 @@ pub fn benchmark_subscribe_speed(c: &mut Criterion) {
let rt = Runtime::new().expect("tokio runtime should start");
c.bench_function(
"subscribe speed: time till pong",
move |b| b.to_async(&rt).iter_custom(|iters| async move {
c.bench_function("subscribe speed: time till pong", move |b| {
b.to_async(&rt).iter_custom(|iters| async move {
// Start a server:
let mut server = start_server(
ServerOpts {
@@ -38,8 +36,9 @@ pub fn benchmark_subscribe_speed(c: &mut Criterion) {
max_node_data_per_second: Some(usize::MAX),
worker_threads: Some(2),
..Default::default()
}
).await;
},
)
.await;
let shard_id = server.add_shard().await.unwrap();
// Connect a shard:
@@ -52,22 +51,24 @@ pub fn benchmark_subscribe_speed(c: &mut Criterion) {
// Add a bunch of actual nodes on the same chain:
for n in 0..NUMBER_OF_NODES {
node_tx.send_json_text(json!({
"id":n,
"ts":"2021-07-12T10:37:47.714666+01:00",
"payload": {
"authority":true,
"chain":"Polkadot", // No limit to #nodes on this network.
"config":"",
"genesis_hash": BlockHash::from_low_u64_ne(1),
"implementation":"Substrate Node",
"msg":"system.connected",
"name": format!("Node {}", n),
"network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
"startup_time":"1625565542717",
"version":"2.0.0-07a1af348-aarch64-macos"
}
})).unwrap();
node_tx
.send_json_text(json!({
"id":n,
"ts":"2021-07-12T10:37:47.714666+01:00",
"payload": {
"authority":true,
"chain":"Polkadot", // No limit to #nodes on this network.
"config":"",
"genesis_hash": BlockHash::from_low_u64_ne(1),
"implementation":"Substrate Node",
"msg":"system.connected",
"name": format!("Node {}", n),
"network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
"startup_time":"1625565542717",
"version":"2.0.0-07a1af348-aarch64-macos"
}
}))
.unwrap();
}
// Give those messages a chance to be handled. This, of course,
@@ -79,7 +80,6 @@ pub fn benchmark_subscribe_speed(c: &mut Criterion) {
// iters performed here, but a lot of the time that number is "1".
let mut total_time = Duration::ZERO;
for _n in 0..iters {
// Start a bunch of feeds:
let mut feeds = server
.get_core()
@@ -94,7 +94,9 @@ pub fn benchmark_subscribe_speed(c: &mut Criterion) {
// Then, Ping a feed:
feeds[0].0.send_command("ping", "Finished!").unwrap();
let finished = FeedMessage::Pong { msg: "Finished!".to_owned() };
let finished = FeedMessage::Pong {
msg: "Finished!".to_owned(),
};
// Wait and see how long it takes to get a pong back:
let start = Instant::now();
@@ -115,7 +117,7 @@ pub fn benchmark_subscribe_speed(c: &mut Criterion) {
// The total time spent waiting for subscribes:
total_time
})
);
});
}
criterion_group!(benches, benchmark_subscribe_speed);
backend/telemetry_core/src/aggregator/aggregator.rs
+13 -8
View File
@@ -113,8 +113,10 @@ impl Aggregator {
/// Return a sink that a feed can send messages into to be handled by the aggregator.
pub fn subscribe_feed(
&self,
) -> (u64, impl Sink<inner_loop::FromFeedWebsocket, Error = anyhow::Error> + Send + Sync + Unpin + 'static)
{
) -> (
u64,
impl Sink<inner_loop::FromFeedWebsocket, Error = anyhow::Error> + Send + Sync + Unpin + 'static,
) {
// Assign a unique aggregator-local ID to each connection that subscribes, and pass
// that along with every message to the aggregator loop:
let feed_conn_id = self
@@ -125,11 +127,14 @@ impl Aggregator {
// Calling `send` on this Sink requires Unpin. There may be a nicer way than this,
// but pinning by boxing is the easy solution for now:
(feed_conn_id, Box::pin(tx_to_aggregator.with(move |msg| async move {
Ok(inner_loop::ToAggregator::FromFeedWebsocket(
feed_conn_id.into(),
msg,
))
})))
(
feed_conn_id,
Box::pin(tx_to_aggregator.with(move |msg| async move {
Ok(inner_loop::ToAggregator::FromFeedWebsocket(
feed_conn_id.into(),
msg,
))
})),
)
}
}
backend/telemetry_core/src/aggregator/inner_loop.rs
+4 -1
View File
@@ -447,7 +447,10 @@ impl InnerLoop {
.chunks(64)
.filter_map(|nodes| {
let mut feed_serializer = FeedMessageSerializer::new();
for (node_id, node) in nodes.iter().filter_map(|&(idx, n)| n.as_ref().map(|n| (idx, n))) {
for (node_id, node) in nodes
.iter()
.filter_map(|&(idx, n)| n.as_ref().map(|n| (idx, n)))
{
feed_serializer.push(feed_message::AddedNode(node_id, node));
feed_serializer.push(feed_message::FinalizedBlock(
node_id,
backend/telemetry_core/src/feed_message.rs
-15
View File
@@ -18,7 +18,6 @@
//! send to subscribed feeds (browsers).
use serde::Serialize;
use std::mem;
use crate::state::Node;
use common::node_types::{
@@ -82,20 +81,6 @@ impl FeedMessageSerializer {
let _ = to_writer(&mut self.buffer, value);
}
/// Return the bytes we've serialized so far and prepare a new buffer. If you're
/// finished serializing data, prefer [`FeedMessageSerializer::into_finalized`]
pub fn finalize(&mut self) -> Option<bytes::Bytes> {
if self.buffer.is_empty() {
return None;
}
self.buffer.push(b']');
let bytes = mem::replace(&mut self.buffer, Vec::with_capacity(BUFCAP));
Some(bytes.into())
}
/// Return the bytes that we've serialized so far, consuming the serializer.
pub fn into_finalized(mut self) -> Option<bytes::Bytes> {
if self.buffer.is_empty() {
backend/telemetry_core/src/main.rs
+2 -2
View File
@@ -77,7 +77,7 @@ fn main() {
let worker_threads = match opts.worker_threads {
0 => num_cpus::get(),
n => n
n => n,
};
tokio::runtime::Builder::new_multi_thread()
@@ -308,7 +308,7 @@ async fn handle_feed_websocket_connection<S>(
mut ws_recv: http_utils::WsReceiver,
mut tx_to_aggregator: S,
feed_timeout: u64,
_feed_id: u64 // <- can be useful for debugging purposes.
_feed_id: u64, // <- can be useful for debugging purposes.
) -> (S, http_utils::WsSender)
where
S: futures::Sink<FromFeedWebsocket, Error = anyhow::Error> + Unpin + Send + 'static,
backend/telemetry_core/tests/e2e_tests.rs
+1 -1
View File
@@ -37,7 +37,7 @@ use std::time::Duration;
use test_utils::{
assert_contains_matches,
feed_message_de::{FeedMessage, NodeDetails},
workspace::{start_server, start_server_debug, ServerOpts, CoreOpts, ShardOpts},
workspace::{start_server, start_server_debug, CoreOpts, ServerOpts, ShardOpts},
};
/// The simplest test we can run; the main benefit of this test (since we check similar)
backend/telemetry_core/tests/soak_tests.rs
+15 -11
View File
@@ -36,13 +36,13 @@ box; MacOS seems to hit limits quicker in general.
use common::node_types::BlockHash;
use common::ws_client::SentMessage;
use futures::{StreamExt, future};
use futures::{future, StreamExt};
use serde_json::json;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
use structopt::StructOpt;
use test_utils::workspace::{start_server, ServerOpts, CoreOpts, ShardOpts};
use test_utils::workspace::{start_server, CoreOpts, ServerOpts, ShardOpts};
/// A configurable soak_test runner. Configure by providing the expected args as
/// an environment variable. One example to run this test is:
@@ -87,7 +87,8 @@ async fn run_soak_test(opts: SoakTestOpts) {
worker_threads: opts.shard_worker_threads,
..Default::default()
},
).await;
)
.await;
println!("Telemetry core running at {}", server.get_core().host());
// Start up the shards we requested:
@@ -274,7 +275,8 @@ async fn run_realistic_soak_test(opts: SoakTestOpts) {
worker_threads: opts.shard_worker_threads,
..Default::default()
},
).await;
)
.await;
println!("Telemetry core running at {}", server.get_core().host());
// Start up the shards we requested:
@@ -307,14 +309,16 @@ async fn run_realistic_soak_test(opts: SoakTestOpts) {
Duration::from_secs(3),
format!("Node {}", idx + 1),
"Polkadot".to_owned(),
idx + 1
idx + 1,
);
let res = telemetry.start(|msg| async {
bytes_in.fetch_add(msg.len(), Ordering::Relaxed);
tx.unbounded_send(SentMessage::Binary(msg))?;
Ok::<_, anyhow::Error>(())
}).await;
let res = telemetry
.start(|msg| async {
bytes_in.fetch_add(msg.len(), Ordering::Relaxed);
tx.unbounded_send(SentMessage::Binary(msg))?;
Ok::<_, anyhow::Error>(())
})
.await;
if let Err(e) = res {
log::error!("Telemetry Node #{} has died with error: {}", idx, e);
@@ -408,7 +412,7 @@ struct SoakTestOpts {
shard_worker_threads: Option<usize>,
/// Should we log output from the core/shards to stdout?
#[structopt(long)]
log_output: bool
log_output: bool,
}
/// Get soak test args from an envvar and parse them via structopt.