Add benchmark machine placeholder (#11198)

* Move new_rng to shared code

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add bechmark machine command

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Use sc-sysinfo

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add --no-hardware-benchmarks

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Lockfile

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Do not create components if not needed

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Fix tests

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Revert "Add --no-hardware-benchmarks"

This reverts commit d4ee98222bf1a5ea62ac60dd7d5c62070e2d7f70.

* Fix tests

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Update Cargo deps

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Move sr255119::verify bench to sc-sysinfo

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Move sr255119::verify bench to sc-sysinfo

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Switch benchmarks to return f64

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* fmt

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Hide command until completed

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Use concrete rand implementation

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Put clobber into a function

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add test

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add comment

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Update cargo to match polkadot

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Remove doc that does not format in the console

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Limit benchmark by time

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add ExecutionLimit and make function infallible

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* CI

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add doc

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

substrate/utils/frame/benchmarking-cli/src/lib.rs

@@ -18,12 +18,14 @@
 //! Contains the root [`BenchmarkCmd`] command and exports its sub-commands.
 
 mod block;
+mod machine;
 mod overhead;
 mod pallet;
 mod shared;
 mod storage;
 
 pub use block::BlockCmd;
+pub use machine::MachineCmd;
 pub use overhead::{ExtrinsicBuilder, OverheadCmd};
 pub use pallet::PalletCmd;
 pub use storage::StorageCmd;
@@ -39,6 +41,8 @@ pub enum BenchmarkCmd {
 	Storage(StorageCmd),
 	Overhead(OverheadCmd),
 	Block(BlockCmd),
+	#[clap(hide = true)] // Hidden until fully completed.
+	Machine(MachineCmd),
 }
 
 /// Unwraps a [`BenchmarkCmd`] into its concrete sub-command.
@@ -53,6 +57,7 @@ macro_rules! unwrap_cmd {
 			BenchmarkCmd::Storage($cmd) => $code,
 			BenchmarkCmd::Overhead($cmd) => $code,
 			BenchmarkCmd::Block($cmd) => $code,
+			BenchmarkCmd::Machine($cmd) => $code,
 		}
 	}
 }

substrate/utils/frame/benchmarking-cli/src/machine/mod.rs

@@ -0,0 +1,86 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Contains the [`MachineCmd`] as entry point for the node
+//! and the core benchmarking logic.
+
+use sc_cli::{CliConfiguration, Result, SharedParams};
+use sc_service::Configuration;
+use sc_sysinfo::{
+	benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes,
+	benchmark_memory, benchmark_sr25519_verify, ExecutionLimit,
+};
+
+use clap::Parser;
+use log::info;
+use prettytable::{cell, row, table};
+use std::{fmt::Debug, fs, time::Duration};
+
+/// Command to benchmark the hardware.
+///
+/// Runs multiple benchmarks and prints their output to console.
+/// Can be used to gauge if the hardware is fast enough to keep up with a chain's requirements.
+/// This command must be integrated by the client since the client can set compiler flags
+/// which influence the results.
+///
+/// You can use the `--base-path` flag to set a location for the disk benchmarks.
+#[derive(Debug, Parser)]
+pub struct MachineCmd {
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub shared_params: SharedParams,
+
+	/// Time limit for the verification benchmark.
+	#[clap(long, default_value = "2.0", value_name = "SECONDS")]
+	pub verify_duration: f32,
+}
+
+impl MachineCmd {
+	/// Execute the benchmark and print the results.
+	pub fn run(&self, cfg: &Configuration) -> Result<()> {
+		// Ensure that the dir exists since the node is not started to take care of it.
+		let dir = cfg.database.path().ok_or("No DB directory provided")?;
+		fs::create_dir_all(dir)?;
+
+		info!("Running machine benchmarks...");
+		let write = benchmark_disk_sequential_writes(dir)?;
+		let read = benchmark_disk_random_writes(dir)?;
+		let verify_limit =
+			ExecutionLimit::MaxDuration(Duration::from_secs_f32(self.verify_duration));
+		let verify = benchmark_sr25519_verify(verify_limit) * 1024.0;
+
+		// Use a table for nicer console output.
+		let table = table!(
+			["Category", "Function", "Score", "Unit"],
+			["CPU", "BLAKE2-256", benchmark_cpu(), "MB/s"],
+			["CPU", "SR25519 Verify", format!("{:.1}", verify), "KB/s"],
+			["Memory", "Copy", benchmark_memory(), "MB/s"],
+			["Disk", "Seq Write", write, "MB/s"],
+			["Disk", "Rnd Write", read, "MB/s"]
+		);
+
+		info!("\n{}", table);
+		Ok(())
+	}
+}
+
+// Boilerplate
+impl CliConfiguration for MachineCmd {
+	fn shared_params(&self) -> &SharedParams {
+		&self.shared_params
+	}
+}

substrate/utils/frame/benchmarking-cli/src/shared/mod.rs

@@ -25,6 +25,8 @@ pub use record::BenchRecord;
 pub use stats::{StatSelect, Stats};
 pub use weight_params::WeightParams;
 
+use rand::prelude::*;
+
 /// A Handlebars helper to add an underscore after every 3rd character,
 /// i.e. a separator for large numbers.
 #[derive(Clone, Copy)]
@@ -63,3 +65,11 @@ where
 	}
 	s
 }
+
+/// Returns an rng and the seed that was used to create it.
+///
+/// Uses a random seed if none is provided.
+pub fn new_rng(seed: Option<u64>) -> (impl rand::Rng, u64) {
+	let seed = seed.unwrap_or(rand::thread_rng().gen::<u64>());
+	(rand_pcg::Pcg64::seed_from_u64(seed), seed)
+}

substrate/utils/frame/benchmarking-cli/src/storage/cmd.rs

@@ -34,7 +34,7 @@ use sp_runtime::generic::BlockId;
 use std::{fmt::Debug, path::PathBuf, sync::Arc};
 
 use super::template::TemplateData;
-use crate::shared::WeightParams;
+use crate::shared::{new_rng, WeightParams};
 
 /// Benchmark the storage speed of a chain snapshot.
 #[derive(Debug, Parser)]
@@ -151,13 +151,6 @@ impl StorageCmd {
 		}
 	}
 
-	/// Creates an rng from a random seed.
-	pub(crate) fn setup_rng() -> impl rand::Rng {
-		let seed = rand::thread_rng().gen::<u64>();
-		info!("Using seed {}", seed);
-		StdRng::seed_from_u64(seed)
-	}
-
 	/// Run some rounds of the (read) benchmark as warmup.
 	/// See `frame_benchmarking_cli::storage::read::bench_read` for detailed comments.
 	fn bench_warmup<B, BA, C>(&self, client: &Arc<C>) -> Result<()>
@@ -169,7 +162,7 @@ impl StorageCmd {
 		let block = BlockId::Number(client.usage_info().chain.best_number);
 		let empty_prefix = StorageKey(Vec::new());
 		let mut keys = client.storage_keys(&block, &empty_prefix)?;
-		let mut rng = Self::setup_rng();
+		let (mut rng, _) = new_rng(None);
 		keys.shuffle(&mut rng);
 
 		for i in 0..self.params.warmups {

substrate/utils/frame/benchmarking-cli/src/storage/read.rs

@@ -28,7 +28,7 @@ use rand::prelude::*;
 use std::{fmt::Debug, sync::Arc, time::Instant};
 
 use super::cmd::StorageCmd;
-use crate::shared::BenchRecord;
+use crate::shared::{new_rng, BenchRecord};
 
 impl StorageCmd {
 	/// Benchmarks the time it takes to read a single Storage item.
@@ -47,7 +47,7 @@ impl StorageCmd {
 		// Load all keys and randomly shuffle them.
 		let empty_prefix = StorageKey(Vec::new());
 		let mut keys = client.storage_keys(&block, &empty_prefix)?;
-		let mut rng = Self::setup_rng();
+		let (mut rng, _) = new_rng(None);
 		keys.shuffle(&mut rng);
 
 		// Interesting part here:

substrate/utils/frame/benchmarking-cli/src/storage/write.rs

@@ -32,7 +32,7 @@ use rand::prelude::*;
 use std::{fmt::Debug, sync::Arc, time::Instant};
 
 use super::cmd::StorageCmd;
-use crate::shared::BenchRecord;
+use crate::shared::{new_rng, BenchRecord};
 
 impl StorageCmd {
 	/// Benchmarks the time it takes to write a single Storage item.
@@ -59,7 +59,7 @@ impl StorageCmd {
 		info!("Preparing keys from block {}", block);
 		// Load all KV pairs and randomly shuffle them.
 		let mut kvs = trie.pairs();
-		let mut rng = Self::setup_rng();
+		let (mut rng, _) = new_rng(None);
 		kvs.shuffle(&mut rng);
 
 		// Generate all random values first; Make sure there are no collisions with existing