Reset benchmarks by removing post-genesis changes (#5435)

* Reset by removing pos-genesis changes

* CLI option for DB cache size

* Update Cargo.lock

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>

substrate/Cargo.lock

@@ -603,9 +603,9 @@ dependencies = [
 
 [[package]]
 name = "clang-sys"
-version = "0.29.2"
+version = "0.29.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f92986241798376849e1a007827041fed9bb36195822c2049d18e174420e0534"
+checksum = "fe6837df1d5cba2397b835c8530f51723267e16abbf83892e9e5af4f0e5dd10a"
 dependencies = [
  "glob 0.3.0",
  "libc",
@@ -4934,9 +4934,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro-hack"
-version = "0.5.14"
+version = "0.5.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcfdefadc3d57ca21cf17990a28ef4c0f7c61383a28cb7604cf4a18e6ede1420"
+checksum = "0d659fe7c6d27f25e9d80a1a094c223f5246f6a6596453e09d7229bf42750b63"
 
 [[package]]
 name = "proc-macro-nested"
@@ -8584,7 +8584,7 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3bfd5b7557925ce778ff9b9ef90e3ade34c524b5ff10e239c69a42d546d2af56"
 dependencies = [
- "rand 0.3.23",
+ "rand 0.7.3",
 ]
 
 [[package]]

substrate/client/db/src/bench.rs

@@ -19,6 +19,7 @@
 use std::sync::Arc;
 use std::path::PathBuf;
 use std::cell::{Cell, RefCell};
+use std::collections::HashMap;
 use rand::Rng;
 
 use hash_db::{Prefix, Hasher};
@@ -54,12 +55,14 @@ pub struct BenchmarkingState<B: BlockT> {
 	genesis_root: B::Hash,
 	state: RefCell<Option<DbState<B>>>,
 	db: Cell<Option<Arc<dyn KeyValueDB>>>,
-	genesis: <DbState<B> as StateBackend<HashFor<B>>>::Transaction,
+	genesis: HashMap<Vec<u8>, (Vec<u8>, i32)>,
+	record: Cell<Vec<Vec<u8>>>,
+	cache_size_mb: Option<usize>,
 }
 
 impl<B: BlockT> BenchmarkingState<B> {
 	/// Create a new instance that creates a database in a temporary dir.
-	pub fn new(genesis: Storage) -> Result<Self, String> {
+	pub fn new(genesis: Storage, cache_size_mb: Option<usize>) -> Result<Self, String> {
 		let temp_dir = PathBuf::from(std::env::temp_dir());
 		let name: String = rand::thread_rng().sample_iter(&rand::distributions::Alphanumeric).take(10).collect();
 		let path = temp_dir.join(&name);
@@ -76,6 +79,8 @@ impl<B: BlockT> BenchmarkingState<B> {
 			root: Cell::new(root),
 			genesis: Default::default(),
 			genesis_root: Default::default(),
+			record: Default::default(),
+			cache_size_mb,
 		};
 
 		state.reopen()?;
@@ -88,7 +93,7 @@ impl<B: BlockT> BenchmarkingState<B> {
 			genesis.top.into_iter().map(|(k, v)| (k, Some(v))),
 			child_delta,
 		);
-		state.genesis = transaction.clone();
+		state.genesis = transaction.clone().drain();
 		state.genesis_root = root.clone();
 		state.commit(root, transaction)?;
 		Ok(state)
@@ -97,7 +102,10 @@ impl<B: BlockT> BenchmarkingState<B> {
 	fn reopen(&self) -> Result<(), String> {
 		*self.state.borrow_mut() = None;
 		self.db.set(None);
-		let db_config = DatabaseConfig::with_columns(1);
+		let mut db_config = DatabaseConfig::with_columns(1);
+		if let Some(size) = &self.cache_size_mb {
+			db_config.memory_budget.insert(0, *size);
+		}
 		let path = self.path.to_str()
 			.ok_or_else(|| String::from("Invalid database path"))?;
 		let db = Arc::new(Database::open(&db_config, &path).map_err(|e| format!("Error opening database: {:?}", e))?);
@@ -257,14 +265,17 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
 	{
 		if let Some(db) = self.db.take() {
 			let mut db_transaction = DBTransaction::new();
-
-			for (key, (val, rc)) in transaction.drain() {
+			let changes = transaction.drain();
+			let mut keys = Vec::with_capacity(changes.len());
+			for (key, (val, rc)) in changes {
 				if rc > 0 {
 					db_transaction.put(0, &key, &val);
 				} else if rc < 0 {
 					db_transaction.delete(0, &key);
 				}
+				keys.push(key);
 			}
+			self.record.set(keys);
 			db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?;
 			self.root.set(storage_root);
 		} else {
@@ -274,9 +285,24 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
 	}
 
 	fn wipe(&self) -> Result<(), Self::Error> {
-		self.kill()?;
+		// Restore to genesis
+		let record = self.record.take();
+		if let Some(db) = self.db.take() {
+			let mut db_transaction = DBTransaction::new();
+			for key in record {
+				match self.genesis.get(&key) {
+					Some((v, _)) => db_transaction.put(0, &key, v),
+					None => db_transaction.delete(0, &key),
+				}
+			}
+			db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?;
+		}
+
+		self.db.set(None);
+		*self.state.borrow_mut() = None;
+
+		self.root.set(self.genesis_root.clone());
 		self.reopen()?;
-		self.commit(self.genesis_root.clone(), self.genesis.clone())?;
 		Ok(())
 	}
 }

substrate/client/service/src/lib.rs

@@ -58,7 +58,7 @@ pub use self::builder::{
 	ServiceBuilder, ServiceBuilderCommand, TFullClient, TLightClient, TFullBackend, TLightBackend,
 	TFullCallExecutor, TLightCallExecutor,
 };
-pub use config::{Configuration, Roles, PruningMode};
+pub use config::{Configuration, Roles, PruningMode, DatabaseConfig};
 pub use sc_chain_spec::{
 	ChainSpec, GenericChainSpec, Properties, RuntimeGenesis, Extension as ChainSpecExtension
 };

substrate/utils/frame/benchmarking-cli/src/lib.rs

@@ -91,6 +91,10 @@ pub struct BenchmarkCmd {
 		default_value = "Interpreted"
 	)]
 	pub wasm_method: WasmExecutionMethod,
+
+	/// Limit the memory the database cache can use.
+	#[structopt(long = "db-cache", value_name = "MiB", default_value = "128")]
+	pub database_cache_size: u32,
 }
 
 impl BenchmarkCmd {
@@ -116,7 +120,8 @@ impl BenchmarkCmd {
 
 		let genesis_storage = spec.build_storage()?;
 		let mut changes = Default::default();
-		let state = BenchmarkingState::<BB>::new(genesis_storage)?;
+		let cache_size = Some(self.database_cache_size as usize);
+		let state = BenchmarkingState::<BB>::new(genesis_storage, cache_size)?;
 		let executor = NativeExecutor::<ExecDispatch>::new(
 			wasm_method,
 			None, // heap pages