From 4a5cd54cd879ad9713fa6ac87cb25d1d5c27faf4 Mon Sep 17 00:00:00 2001 From: James Wilson Date: Thu, 28 Aug 2025 14:52:08 +0100 Subject: [PATCH] Backend: add parallel cpu benchmarks processing (#603) * Add parallel cpu benchmarks processing * Expose Parallel Hashrate Score in UI (as 'Parallel CPU speed') * Remove unused variable and fix cfg(debug) * Fix tests after removing old_chain_name * Update Dockerfile * bullseye for both images --------- Co-authored-by: Volodymyr Brazhnyk --- backend/Dockerfile | 6 +++--- backend/common/src/node_message.rs | 3 +++ backend/common/src/node_types.rs | 5 +++++ backend/telemetry_core/src/feed_message.rs | 1 + backend/telemetry_core/src/state/chain.rs | 1 + backend/telemetry_core/src/state/chain_stats.rs | 10 ++++++++++ backend/telemetry_core/src/state/state.rs | 13 ++----------- .../src/json_message/node_message.rs | 5 +++++ backend/telemetry_shard/src/main.rs | 4 ++-- frontend/src/common/types.ts | 1 + frontend/src/components/Stats/Stats.tsx | 6 ++++++ 11 files changed, 39 insertions(+), 16 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index f98c1ed..4c0d343 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,14 +1,14 @@ -FROM docker.io/paritytech/ci-linux:production as builder +FROM docker.io/paritytech/ci-unified:bullseye-1.88.0-2025-06-27-v202507221446 AS builder ARG PROFILE=release WORKDIR /app COPY . . -RUN cargo build --${PROFILE} --bins +RUN cargo build --${PROFILE} --bins -j 2 # MAIN IMAGE FOR PEOPLE TO PULL --- small one# -FROM docker.io/debian:buster-slim +FROM docker.io/debian:bullseye-slim LABEL maintainer="Parity Technologies" LABEL description="Substrate Telemetry Backend shard/core binaries, static build" diff --git a/backend/common/src/node_message.rs b/backend/common/src/node_message.rs index 9b63e8b..ceb32a0 100644 --- a/backend/common/src/node_message.rs +++ b/backend/common/src/node_message.rs @@ -98,6 +98,9 @@ pub struct NodeHwBench { pub memory_memcpy_score: u64, pub disk_sequential_write_score: Option, pub disk_random_write_score: Option, + pub parallel_cpu_hashrate_score: Option, + //// Dev note: this exists but isn't needed yet: + // pub parallel_cpu_cores: Option, } impl Payload { diff --git a/backend/common/src/node_types.rs b/backend/common/src/node_types.rs index 5cafd1b..ac05fbd 100644 --- a/backend/common/src/node_types.rs +++ b/backend/common/src/node_types.rs @@ -73,6 +73,11 @@ pub struct NodeHwBench { pub disk_sequential_write_score: Option, /// Random disk write speed in MB/s. pub disk_random_write_score: Option, + /// The parallel CPU speed, as measured in how many MB/s it can hash using the BLAKE2b-256 hash. + pub parallel_cpu_hashrate_score: Option, + //// Dev note: this exists but isn't needed yet: + // /// The number of cores used for the parallel CPU benchmark. + // pub parallel_cpu_cores: Option, } /// A couple of node statistics. diff --git a/backend/telemetry_core/src/feed_message.rs b/backend/telemetry_core/src/feed_message.rs index 2ba2a91..00f94a2 100644 --- a/backend/telemetry_core/src/feed_message.rs +++ b/backend/telemetry_core/src/feed_message.rs @@ -246,6 +246,7 @@ pub struct ChainStats { pub linux_distro: Ranking, pub is_virtual_machine: Ranking, pub cpu_hashrate_score: Ranking<(u32, Option)>, + pub parallel_cpu_hashrate_score: Ranking<(u32, Option)>, pub memory_memcpy_score: Ranking<(u32, Option)>, pub disk_sequential_write_score: Ranking<(u32, Option)>, pub disk_random_write_score: Ranking<(u32, Option)>, diff --git a/backend/telemetry_core/src/state/chain.rs b/backend/telemetry_core/src/state/chain.rs index 8e265a8..60f1613 100644 --- a/backend/telemetry_core/src/state/chain.rs +++ b/backend/telemetry_core/src/state/chain.rs @@ -213,6 +213,7 @@ impl Chain { memory_memcpy_score: hwbench.memory_memcpy_score, disk_sequential_write_score: hwbench.disk_sequential_write_score, disk_random_write_score: hwbench.disk_random_write_score, + parallel_cpu_hashrate_score: hwbench.parallel_cpu_hashrate_score, }; let old_hwbench = node.update_hwbench(new_hwbench); // The `hwbench` for this node has changed, send an updated "add node". diff --git a/backend/telemetry_core/src/state/chain_stats.rs b/backend/telemetry_core/src/state/chain_stats.rs index be5c946..ec04f00 100644 --- a/backend/telemetry_core/src/state/chain_stats.rs +++ b/backend/telemetry_core/src/state/chain_stats.rs @@ -149,6 +149,7 @@ pub struct ChainStatsCollator { disk_sequential_write_score: Counter<(u32, Option)>, disk_random_write_score: Counter<(u32, Option)>, cpu_vendor: Counter, + parallel_cpu_hashrate_score: Counter<(u32, Option)>, } impl ChainStatsCollator { @@ -241,6 +242,14 @@ impl ChainStatsCollator { .as_ref(), op, ); + + self.parallel_cpu_hashrate_score.modify( + hwbench + .and_then(|hwbench| hwbench.parallel_cpu_hashrate_score) + .map(|score| bucket_score(score, REFERENCE_CPU_SCORE)) + .as_ref(), + op, + ); } pub fn generate(&self) -> ChainStats { @@ -261,6 +270,7 @@ impl ChainStatsCollator { .generate_ranking_ordered(), disk_random_write_score: self.disk_random_write_score.generate_ranking_ordered(), cpu_vendor: self.cpu_vendor.generate_ranking_top(10), + parallel_cpu_hashrate_score: self.parallel_cpu_hashrate_score.generate_ranking_top(10), } } } diff --git a/backend/telemetry_core/src/state/state.rs b/backend/telemetry_core/src/state/state.rs index f34da06..d256a88 100644 --- a/backend/telemetry_core/src/state/state.rs +++ b/backend/telemetry_core/src/state/state.rs @@ -79,8 +79,6 @@ impl<'a> AddNodeResult<'a> { pub struct NodeAddedToChain<'a> { /// The ID assigned to this node. pub id: NodeId, - /// The old label of the chain. - pub old_chain_label: Box, /// The new label of the chain. pub new_chain_label: &'a str, /// The node that was added. @@ -97,8 +95,6 @@ pub struct RemovedNode { pub chain_node_count: usize, /// Has the chain label been updated? pub has_chain_label_changed: bool, - /// The old label of the chain. - pub old_chain_label: Box, /// Genesis hash of the chain to be updated. pub chain_genesis_hash: BlockHash, /// The new label of the chain. @@ -164,7 +160,6 @@ impl State { ); let node = Node::new(node_details); - let old_chain_label = chain.label().into(); match chain.add_node(node) { chain::AddNodeResult::Overquota => AddNodeResult::ChainOverQuota, @@ -174,7 +169,6 @@ impl State { AddNodeResult::NodeAddedToChain(NodeAddedToChain { id: NodeId(chain_id, id), node: chain.get_node(id).expect("node added above"), - old_chain_label, new_chain_label: chain.label(), chain_node_count: chain.node_count(), has_chain_label_changed: chain_renamed, @@ -186,7 +180,6 @@ impl State { /// Remove a node pub fn remove_node(&mut self, NodeId(chain_id, chain_node_id): NodeId) -> Option { let chain = self.chains.get_mut(chain_id)?; - let old_chain_label = chain.label().into(); // Actually remove the node let remove_result = chain.remove_node(chain_node_id); @@ -204,7 +197,6 @@ impl State { } Some(RemovedNode { - old_chain_label, new_chain_label, chain_node_count, chain_genesis_hash, @@ -320,12 +312,11 @@ mod test { }; assert_eq!(add_node_result.id, NodeId(0.into(), 0.into())); - assert_eq!(&*add_node_result.old_chain_label, ""); assert_eq!(&*add_node_result.new_chain_label, "Chain One"); assert_eq!(add_node_result.chain_node_count, 1); assert_eq!(add_node_result.has_chain_label_changed, true); - let add_result = state.add_node(chain1_genesis, node("A", "Chain One")); + let add_result = state.add_node(chain1_genesis, node("A", "Chain Two")); let add_node_result = match add_result { AddNodeResult::ChainOnDenyList => panic!("Chain not on deny list"), @@ -334,7 +325,7 @@ mod test { }; assert_eq!(add_node_result.id, NodeId(0.into(), 1.into())); - assert_eq!(&*add_node_result.old_chain_label, "Chain One"); + // Chain One and Chain Two as common, so Chain One is kept. assert_eq!(&*add_node_result.new_chain_label, "Chain One"); assert_eq!(add_node_result.chain_node_count, 2); assert_eq!(add_node_result.has_chain_label_changed, false); diff --git a/backend/telemetry_shard/src/json_message/node_message.rs b/backend/telemetry_shard/src/json_message/node_message.rs index f0abb18..3c4ca93 100644 --- a/backend/telemetry_shard/src/json_message/node_message.rs +++ b/backend/telemetry_shard/src/json_message/node_message.rs @@ -211,6 +211,9 @@ pub struct NodeHwBench { pub memory_memcpy_score: u64, pub disk_sequential_write_score: Option, pub disk_random_write_score: Option, + pub parallel_cpu_hashrate_score: Option, + //// This exists but isn't needed at the moment: + // pub parallel_cpu_cores: Option, } impl From for node_types::NodeHwBench { @@ -220,6 +223,7 @@ impl From for node_types::NodeHwBench { memory_memcpy_score: hwbench.memory_memcpy_score, disk_sequential_write_score: hwbench.disk_sequential_write_score, disk_random_write_score: hwbench.disk_random_write_score, + parallel_cpu_hashrate_score: hwbench.parallel_cpu_hashrate_score, } } } @@ -231,6 +235,7 @@ impl From for internal::NodeHwBench { memory_memcpy_score: msg.memory_memcpy_score, disk_sequential_write_score: msg.disk_sequential_write_score, disk_random_write_score: msg.disk_random_write_score, + parallel_cpu_hashrate_score: msg.parallel_cpu_hashrate_score, } } } diff --git a/backend/telemetry_shard/src/main.rs b/backend/telemetry_shard/src/main.rs index 3b30728..b0fb800 100644 --- a/backend/telemetry_shard/src/main.rs +++ b/backend/telemetry_shard/src/main.rs @@ -323,14 +323,14 @@ where // Deserialize from JSON, warning in debug mode if deserialization fails: let node_message: json_message::NodeMessage = match serde_json::from_slice(&bytes) { Ok(node_message) => node_message, - #[cfg(debug)] + #[cfg(debug_assertions)] Err(e) => { let bytes: &[u8] = bytes.get(..512).unwrap_or_else(|| &bytes); let msg_start = std::str::from_utf8(bytes).unwrap_or_else(|_| "INVALID UTF8"); log::warn!("Failed to parse node message ({msg_start}): {e}"); continue; }, - #[cfg(not(debug))] + #[cfg(not(debug_assertions))] Err(_) => { continue; } diff --git a/frontend/src/common/types.ts b/frontend/src/common/types.ts index 211480a..fa3e7de 100644 --- a/frontend/src/common/types.ts +++ b/frontend/src/common/types.ts @@ -131,6 +131,7 @@ export type ChainStats = { linux_distro: Maybe>; linux_kernel: Maybe>; cpu_hashrate_score: Maybe>; + parallel_cpu_hashrate_score: Maybe>; memory_memcpy_score: Maybe>; disk_sequential_write_score: Maybe>; disk_random_write_score: Maybe>; diff --git a/frontend/src/components/Stats/Stats.tsx b/frontend/src/components/Stats/Stats.tsx index 7fee947..24d7397 100644 --- a/frontend/src/components/Stats/Stats.tsx +++ b/frontend/src/components/Stats/Stats.tsx @@ -174,6 +174,12 @@ export class Stats extends React.Component { formatScore, stats.cpu_hashrate_score ); + add( + 'parallel_cpi_hashrate_score', + 'Parallel CPU Speed', + formatScore, + stats.parallel_cpu_hashrate_score + ); add( 'memory_memcpy_score', 'Memory Speed',