Follow ups for benchmark machine (#11270)

* Follow ups for the MachineCmd

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Fix CI

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add to node-template

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Fix test with feature flag

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Lower disk requirements

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add ExecutionLimit to the disk benchmarks

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* fmt

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add doc

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Rename DISK_WRITE_LIMIT -> DEFAULT_DISK_EXECUTION_LIMIT

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Rename POLKADOT_REFERENCE_HARDWARE -> SUBSTRATE_REFERENCE_HARDWARE

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Fix build profile + add license

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Remove deps

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Set tolerance to 10%

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Fix tests

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Ignore test

I cannot reproduce the CI error, even with the full command:
cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path ./bin/node/cli/Cargo.toml

I will put an 'ignore' on that test for now, since it works for me and is worth having.

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Remove test

Still cannot reproduce the error and it fails in the CI.
Removing it now.

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>

substrate/bin/node/cli/Cargo.toml

@@ -96,6 +96,7 @@ sc-cli = { version = "0.10.0-dev", optional = true, path = "../../../client/cli"
 frame-benchmarking-cli = { version = "4.0.0-dev", optional = true, path = "../../../utils/frame/benchmarking-cli" }
 node-inspect = { version = "0.9.0-dev", optional = true, path = "../inspect" }
 try-runtime-cli = { version = "0.10.0-dev", optional = true, path = "../../../utils/frame/try-runtime/cli" }
+serde_json = "1.0.79"
 
 [target.'cfg(any(target_arch="x86_64", target_arch="aarch64"))'.dependencies]
 node-executor = { version = "3.0.0-dev", path = "../executor", features = ["wasmtime"] }

substrate/bin/node/cli/src/command.rs

@@ -130,7 +130,8 @@ pub fn run() -> Result<()> {
 
 						cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder))
 					},
-					BenchmarkCmd::Machine(cmd) => cmd.run(&config),
+					BenchmarkCmd::Machine(cmd) =>
+						cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()),
 				}
 			})
 		},

substrate/bin/node/cli/tests/benchmark_machine_works.rs

@@ -24,9 +24,30 @@ use std::process::Command;
 fn benchmark_machine_works() {
 	let status = Command::new(cargo_bin("substrate"))
 		.args(["benchmark", "machine", "--dev"])
-		.args(["--verify-duration", "0.1"])
+		.args(["--verify-duration", "0.1", "--disk-duration", "0.1"])
+		// Make it succeed.
+		.args(["--allow-fail"])
 		.status()
 		.unwrap();
 
 	assert!(status.success());
 }
+
+/// Test that the hardware does not meet the requirements.
+///
+/// This is most likely to succeed since it uses a test profile.
+#[test]
+#[cfg(debug_assertions)]
+fn benchmark_machine_fails_with_slow_hardware() {
+	let output = Command::new(cargo_bin("substrate"))
+		.args(["benchmark", "machine", "--dev"])
+		.args(["--verify-duration", "0.1", "--disk-duration", "2", "--tolerance", "0"])
+		.output()
+		.unwrap();
+
+	// Command should have failed.
+	assert!(!output.status.success());
+	// An `UnmetRequirement` error should have been printed.
+	let log = String::from_utf8_lossy(&output.stderr).to_string();
+	assert!(log.contains("UnmetRequirement"));
+}