Add benchmark machine placeholder (#11198)

* Move new_rng to shared code Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add bechmark machine command Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Use sc-sysinfo Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add --no-hardware-benchmarks Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Lockfile Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Do not create components if not needed Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Fix tests Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Revert "Add --no-hardware-benchmarks" This reverts commit d4ee98222bf1a5ea62ac60dd7d5c62070e2d7f70. * Fix tests Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Update Cargo deps Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Move sr255119::verify bench to sc-sysinfo Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Move sr255119::verify bench to sc-sysinfo Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Switch benchmarks to return f64 Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Review fixes Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * fmt Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Hide command until completed Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Use concrete rand implementation Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Put clobber into a function Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add test Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add comment Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Update cargo to match polkadot Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Remove doc that does not format in the console Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Limit benchmark by time Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add ExecutionLimit and make function infallible Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * CI Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add doc Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>
2026-04-26 09:57:56 +00:00 · 2022-04-15 14:09:16 +02:00
parent ae75a371bf
commit 362dc50940
15 changed files with 351 additions and 46 deletions
@@ -23,6 +23,7 @@ sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" }
 sc-cli = { version = "0.10.0-dev", path = "../../../client/cli" }
 sc-client-db = { version = "0.10.0-dev", path = "../../../client/db" }
 sc-executor = { version = "0.10.0-dev", path = "../../../client/executor" }
+sc-sysinfo = { version = "6.0.0-dev", path = "../../../client/sysinfo" }

 sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" }
 sp-externalities = { version = "0.12.0", path = "../../../primitives/externalities" }
@@ -32,7 +33,6 @@ sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" }
 sp-keystore = { version = "0.12.0", path = "../../../primitives/keystore" }
 sp-storage = { version = "6.0.0", path = "../../../primitives/storage" }
 sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" }
-sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" }
 sp-state-machine = { version = "0.12.0", path = "../../../primitives/state-machine" }
 sp-trie = { version = "6.0.0", path = "../../../primitives/trie" }
 codec = { version = "3.0.0", package = "parity-scale-codec" }
@@ -52,6 +52,9 @@ hex = "0.4.3"
 memory-db = "0.29.0"
 rand = { version = "0.8.4", features = ["small_rng"] }
 thousands = "0.2.0"
+prettytable-rs = "0.8.0"
+tempfile = "3.2.0"
+rand_pcg = "0.3.1"

 [features]
 default = ["db", "sc-client-db/runtime-benchmarks"]
@@ -18,12 +18,14 @@
 //! Contains the root [`BenchmarkCmd`] command and exports its sub-commands.

 mod block;
+mod machine;
 mod overhead;
 mod pallet;
 mod shared;
 mod storage;

 pub use block::BlockCmd;
+pub use machine::MachineCmd;
 pub use overhead::{ExtrinsicBuilder, OverheadCmd};
 pub use pallet::PalletCmd;
 pub use storage::StorageCmd;
@@ -39,6 +41,8 @@ pub enum BenchmarkCmd {
 	Storage(StorageCmd),
 	Overhead(OverheadCmd),
 	Block(BlockCmd),
+	#[clap(hide = true)] // Hidden until fully completed.
+	Machine(MachineCmd),
 }

 /// Unwraps a [`BenchmarkCmd`] into its concrete sub-command.
@@ -53,6 +57,7 @@ macro_rules! unwrap_cmd {
 			BenchmarkCmd::Storage($cmd) => $code,
 			BenchmarkCmd::Overhead($cmd) => $code,
 			BenchmarkCmd::Block($cmd) => $code,
+			BenchmarkCmd::Machine($cmd) => $code,
 		}
 	}
 }
@@ -0,0 +1,86 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Contains the [`MachineCmd`] as entry point for the node
+//! and the core benchmarking logic.
+
+use sc_cli::{CliConfiguration, Result, SharedParams};
+use sc_service::Configuration;
+use sc_sysinfo::{
+	benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes,
+	benchmark_memory, benchmark_sr25519_verify, ExecutionLimit,
+};
+
+use clap::Parser;
+use log::info;
+use prettytable::{cell, row, table};
+use std::{fmt::Debug, fs, time::Duration};
+
+/// Command to benchmark the hardware.
+///
+/// Runs multiple benchmarks and prints their output to console.
+/// Can be used to gauge if the hardware is fast enough to keep up with a chain's requirements.
+/// This command must be integrated by the client since the client can set compiler flags
+/// which influence the results.
+///
+/// You can use the `--base-path` flag to set a location for the disk benchmarks.
+#[derive(Debug, Parser)]
+pub struct MachineCmd {
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub shared_params: SharedParams,
+
+	/// Time limit for the verification benchmark.
+	#[clap(long, default_value = "2.0", value_name = "SECONDS")]
+	pub verify_duration: f32,
+}
+
+impl MachineCmd {
+	/// Execute the benchmark and print the results.
+	pub fn run(&self, cfg: &Configuration) -> Result<()> {
+		// Ensure that the dir exists since the node is not started to take care of it.
+		let dir = cfg.database.path().ok_or("No DB directory provided")?;
+		fs::create_dir_all(dir)?;
+
+		info!("Running machine benchmarks...");
+		let write = benchmark_disk_sequential_writes(dir)?;
+		let read = benchmark_disk_random_writes(dir)?;
+		let verify_limit =
+			ExecutionLimit::MaxDuration(Duration::from_secs_f32(self.verify_duration));
+		let verify = benchmark_sr25519_verify(verify_limit) * 1024.0;
+
+		// Use a table for nicer console output.
+		let table = table!(
+			["Category", "Function", "Score", "Unit"],
+			["CPU", "BLAKE2-256", benchmark_cpu(), "MB/s"],
+			["CPU", "SR25519 Verify", format!("{:.1}", verify), "KB/s"],
+			["Memory", "Copy", benchmark_memory(), "MB/s"],
+			["Disk", "Seq Write", write, "MB/s"],
+			["Disk", "Rnd Write", read, "MB/s"]
+		);
+
+		info!("\n{}", table);
+		Ok(())
+	}
+}
+
+// Boilerplate
+impl CliConfiguration for MachineCmd {
+	fn shared_params(&self) -> &SharedParams {
+		&self.shared_params
+	}
+}
@@ -25,6 +25,8 @@ pub use record::BenchRecord;
 pub use stats::{StatSelect, Stats};
 pub use weight_params::WeightParams;

+use rand::prelude::*;
+
 /// A Handlebars helper to add an underscore after every 3rd character,
 /// i.e. a separator for large numbers.
 #[derive(Clone, Copy)]
@@ -63,3 +65,11 @@ where
 	}
 	s
 }
+
+/// Returns an rng and the seed that was used to create it.
+///
+/// Uses a random seed if none is provided.
+pub fn new_rng(seed: Option<u64>) -> (impl rand::Rng, u64) {
+	let seed = seed.unwrap_or(rand::thread_rng().gen::<u64>());
+	(rand_pcg::Pcg64::seed_from_u64(seed), seed)
+}
@@ -34,7 +34,7 @@ use sp_runtime::generic::BlockId;
 use std::{fmt::Debug, path::PathBuf, sync::Arc};

 use super::template::TemplateData;
-use crate::shared::WeightParams;
+use crate::shared::{new_rng, WeightParams};

 /// Benchmark the storage speed of a chain snapshot.
 #[derive(Debug, Parser)]
@@ -151,13 +151,6 @@ impl StorageCmd {
 		}
 	}

-	/// Creates an rng from a random seed.
-	pub(crate) fn setup_rng() -> impl rand::Rng {
-		let seed = rand::thread_rng().gen::<u64>();
-		info!("Using seed {}", seed);
-		StdRng::seed_from_u64(seed)
-	}
-
 	/// Run some rounds of the (read) benchmark as warmup.
 	/// See `frame_benchmarking_cli::storage::read::bench_read` for detailed comments.
 	fn bench_warmup<B, BA, C>(&self, client: &Arc<C>) -> Result<()>
@@ -169,7 +162,7 @@ impl StorageCmd {
 		let block = BlockId::Number(client.usage_info().chain.best_number);
 		let empty_prefix = StorageKey(Vec::new());
 		let mut keys = client.storage_keys(&block, &empty_prefix)?;
-		let mut rng = Self::setup_rng();
+		let (mut rng, _) = new_rng(None);
 		keys.shuffle(&mut rng);

 		for i in 0..self.params.warmups {
@@ -28,7 +28,7 @@ use rand::prelude::*;
 use std::{fmt::Debug, sync::Arc, time::Instant};

 use super::cmd::StorageCmd;
-use crate::shared::BenchRecord;
+use crate::shared::{new_rng, BenchRecord};

 impl StorageCmd {
 	/// Benchmarks the time it takes to read a single Storage item.
@@ -47,7 +47,7 @@ impl StorageCmd {
 		// Load all keys and randomly shuffle them.
 		let empty_prefix = StorageKey(Vec::new());
 		let mut keys = client.storage_keys(&block, &empty_prefix)?;
-		let mut rng = Self::setup_rng();
+		let (mut rng, _) = new_rng(None);
 		keys.shuffle(&mut rng);

 		// Interesting part here:
@@ -32,7 +32,7 @@ use rand::prelude::*;
 use std::{fmt::Debug, sync::Arc, time::Instant};

 use super::cmd::StorageCmd;
-use crate::shared::BenchRecord;
+use crate::shared::{new_rng, BenchRecord};

 impl StorageCmd {
 	/// Benchmarks the time it takes to write a single Storage item.
@@ -59,7 +59,7 @@ impl StorageCmd {
 		info!("Preparing keys from block {}", block);
 		// Load all KV pairs and randomly shuffle them.
 		let mut kvs = trie.pairs();
-		let mut rng = Self::setup_rng();
+		let (mut rng, _) = new_rng(None);
 		kvs.shuffle(&mut rng);

 		// Generate all random values first; Make sure there are no collisions with existing