node-bench no-op block import (#5869)

* start to try and implement noop

* txs as input

* better comment

* Add transfer reaping

* rename to avoid filter matching

* Update base weights based on results

* fix priority

* fix logic on reaping transfer

* Update bin/node/bench/src/import.rs

Co-authored-by: Nikolay Volf <nikvolf@gmail.com>

* Update bin/node/bench/src/main.rs

Co-authored-by: Nikolay Volf <nikvolf@gmail.com>

* add back size type (in-progress)

* bring back size type with custom

* update comment

* nit

* block type then size

* Use `transfer_keep_alive`

Co-authored-by: Nikolay Volf <nikvolf@gmail.com>

substrate/bin/node/bench/src/import.rs

@@ -49,16 +49,20 @@ pub enum SizeType {
 	Large,
 	#[display(fmt = "full")]
 	Full,
+	#[display(fmt = "custom")]
+	Custom,
 }
 
 impl SizeType {
-	fn transactions(&self) -> usize {
+	pub fn transactions(&self) -> usize {
 		match self {
 			SizeType::Empty => 0,
 			SizeType::Small => 10,
 			SizeType::Medium => 100,
 			SizeType::Large => 500,
 			SizeType::Full => 4000,
+			// Custom SizeType will use the `--transactions` input parameter
+			SizeType::Custom => 0,
 		}
 	}
 }
@@ -66,6 +70,7 @@ impl SizeType {
 pub struct ImportBenchmarkDescription {
 	pub profile: Profile,
 	pub key_types: KeyTypes,
+	pub block_type: BlockType,
 	pub size: SizeType,
 }
 
@@ -90,6 +95,12 @@ impl core::BenchmarkDescription for ImportBenchmarkDescription {
 			KeyTypes::Ed25519 => path.push("ed25519"),
 		}
 
+		match self.block_type {
+			BlockType::RandomTransfersKeepAlive(_) => path.push("transfer_keep_alive"),
+			BlockType::RandomTransfersReaping(_) => path.push("transfer_reaping"),
+			BlockType::Noop(_) => path.push("noop"),
+		}
+
 		path.push(&format!("{}", self.size));
 
 		path
@@ -101,7 +112,7 @@ impl core::BenchmarkDescription for ImportBenchmarkDescription {
 			50_000,
 			self.key_types
 		);
-		let block = bench_db.generate_block(BlockType::RandomTransfers(self.size.transactions()));
+		let block = bench_db.generate_block(self.block_type);
 		Box::new(ImportBenchmark {
 			database: bench_db,
 			block,
@@ -110,16 +121,11 @@ impl core::BenchmarkDescription for ImportBenchmarkDescription {
 	}
 
 	fn name(&self) -> Cow<'static, str> {
-		match self.profile {
-			Profile::Wasm => format!(
-				"Import benchmark (random transfers, wasm, {} block)",
-				self.size,
-			).into(),
-			Profile::Native => format!(
-				"Import benchmark (random transfers, native, {} block)",
-				self.size,
-			).into(),
-		}
+		format!(
+			"Import benchmark ({:?}, {:?})",
+			self.block_type,
+			self.profile,
+		).into()
 	}
 }
 
@@ -159,4 +165,4 @@ impl core::Benchmark for ImportBenchmark {
 
 		elapsed
 	}
-}
+}

substrate/bin/node/bench/src/main.rs

@@ -26,7 +26,7 @@ use crate::core::{run_benchmark, Mode as BenchmarkMode};
 use crate::tempdb::DatabaseType;
 use import::{ImportBenchmarkDescription, SizeType};
 use trie::{TrieReadBenchmarkDescription, TrieWriteBenchmarkDescription, DatabaseSize};
-use node_testing::bench::{Profile, KeyTypes};
+use node_testing::bench::{Profile, KeyTypes, BlockType};
 use structopt::StructOpt;
 
 #[derive(Debug, StructOpt)]
@@ -49,9 +49,13 @@ struct Opt {
 	/// Run with `--list` for the hint of what to filter.
 	filter: Option<String>,
 
+	/// Number of transactions for block import with `custom` size.
+	#[structopt(long)]
+	transactions: Option<usize>,
+
 	/// Mode
 	///
-	/// "regular" for regular becnhmark
+	/// "regular" for regular benchmark
 	///
 	/// "profile" mode adds pauses between measurable runs,
 	/// so that actual interval can be selected in the profiler of choice.
@@ -66,38 +70,38 @@ fn main() {
 		sc_cli::init_logger("");
 	}
 
+	let mut import_benchmarks = Vec::new();
+
+	for profile in [Profile::Wasm, Profile::Native].iter() {
+		for size in [
+			SizeType::Empty,
+			SizeType::Small,
+			SizeType::Medium,
+			SizeType::Large,
+			SizeType::Full,
+			SizeType::Custom,
+		].iter() {
+			let txs = match size {
+				SizeType::Custom => opt.transactions.unwrap_or(0),
+				_ => size.transactions()
+			};
+			for block_type in [
+				BlockType::RandomTransfersKeepAlive(txs),
+				BlockType::RandomTransfersReaping(txs),
+				BlockType::Noop(txs),
+			].iter() {
+				import_benchmarks.push((profile.clone(), size.clone(), block_type.clone()));
+			}
+		}
+	}
+
 	let benchmarks = matrix!(
-		profile in [Profile::Wasm, Profile::Native].iter() =>
+		(profile, size, block_type) in import_benchmarks.iter() =>
 			ImportBenchmarkDescription {
 				profile: *profile,
 				key_types: KeyTypes::Sr25519,
-				size: SizeType::Medium,
-			},
-		ImportBenchmarkDescription {
-			profile: Profile::Wasm,
-			key_types: KeyTypes::Sr25519,
-			size: SizeType::Empty,
-		},
-		ImportBenchmarkDescription {
-			profile: Profile::Native,
-			key_types: KeyTypes::Ed25519,
-			size: SizeType::Medium,
-		},
-		ImportBenchmarkDescription {
-			profile: Profile::Wasm,
-			key_types: KeyTypes::Sr25519,
-			size: SizeType::Full,
-		},
-		ImportBenchmarkDescription {
-			profile: Profile::Native,
-			key_types: KeyTypes::Sr25519,
-			size: SizeType::Full,
-		},
-		size in [SizeType::Small, SizeType::Large].iter() =>
-			ImportBenchmarkDescription {
-				profile: Profile::Native,
-				key_types: KeyTypes::Sr25519,
 				size: *size,
+				block_type: *block_type,
 			},
 		(size, db_type) in
 			[
@@ -145,4 +149,4 @@ fn main() {
 		let json_result: String = serde_json::to_string(&results).expect("Failed to construct json");
 		println!("{}", json_result);
 	}
-}
+}