Benchmarks now use in-memory db & cache (#5586)

* in-mem state for benchmarks

* Use caching state

* Update Cargo.lock

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>

substrate/Cargo.lock

@@ -1239,7 +1239,7 @@ dependencies = [
  "fixed-hash",
  "impl-rlp",
  "impl-serde 0.3.0",
- "tiny-keccak 2.0.1",
+ "tiny-keccak 2.0.2",
 ]
 
 [[package]]
@@ -1974,7 +1974,7 @@ dependencies = [
  "indexmap",
  "log",
  "slab",
- "tokio 0.2.13",
+ "tokio 0.2.16",
  "tokio-util",
 ]
 
@@ -2186,7 +2186,7 @@ dependencies = [
  "net2",
  "pin-project",
  "time",
- "tokio 0.2.13",
+ "tokio 0.2.16",
  "tower-service",
  "want 0.3.0",
 ]
@@ -2204,7 +2204,7 @@ dependencies = [
  "log",
  "rustls 0.17.0",
  "rustls-native-certs",
- "tokio 0.2.13",
+ "tokio 0.2.16",
  "tokio-rustls",
  "webpki",
 ]
@@ -4873,9 +4873,9 @@ dependencies = [
 
 [[package]]
 name = "paste"
-version = "0.1.9"
+version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "092d791bf7847f70bbd49085489fba25fc2c193571752bff9e36e74e72403932"
+checksum = "ab4fb1930692d1b6a9cfabdde3d06ea0a7d186518e2f4d67660d8970e2fa647a"
 dependencies = [
  "paste-impl",
  "proc-macro-hack",
@@ -4883,9 +4883,9 @@ dependencies = [
 
 [[package]]
 name = "paste-impl"
-version = "0.1.9"
+version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "406c23fb4c45cc6f68a9bbabb8ec7bd6f8cfcbd17e9e8f72c2460282f8325729"
+checksum = "a62486e111e571b1e93b710b61e8f493c0013be39629b714cb166bdb06aa5a8a"
 dependencies = [
  "proc-macro-hack",
  "proc-macro2",
@@ -5229,9 +5229,9 @@ dependencies = [
 
 [[package]]
 name = "quicksink"
-version = "0.1.1"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8461ef7445f61fd72d8dcd0629ce724b9131b3c2eb36e83a5d3d4161c127530"
+checksum = "77de3c815e5a160b1539c6592796801df2043ae35e123b46d73380cfa57af858"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -5895,7 +5895,7 @@ dependencies = [
  "substrate-prometheus-endpoint",
  "tempfile",
  "time",
- "tokio 0.2.13",
+ "tokio 0.2.16",
 ]
 
 [[package]]
@@ -5986,7 +5986,6 @@ dependencies = [
  "parity-util-mem",
  "parking_lot 0.10.0",
  "quickcheck",
- "rand 0.7.3",
  "sc-client",
  "sc-client-api",
  "sc-executor",
@@ -6154,7 +6153,7 @@ dependencies = [
  "substrate-test-runtime-client",
  "substrate-test-runtime-transaction-pool",
  "tempfile",
- "tokio 0.2.13",
+ "tokio 0.2.16",
 ]
 
 [[package]]
@@ -6336,7 +6335,7 @@ dependencies = [
  "substrate-prometheus-endpoint",
  "substrate-test-runtime-client",
  "tempfile",
- "tokio 0.2.13",
+ "tokio 0.2.16",
 ]
 
 [[package]]
@@ -6501,7 +6500,7 @@ dependencies = [
  "sp-utils",
  "substrate-test-runtime-client",
  "threadpool",
- "tokio 0.2.13",
+ "tokio 0.2.16",
 ]
 
 [[package]]
@@ -6919,18 +6918,18 @@ checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0"
 
 [[package]]
 name = "serde"
-version = "1.0.105"
+version = "1.0.106"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e707fbbf255b8fc8c3b99abb91e7257a622caeb20a9818cbadbeeede4e0932ff"
+checksum = "36df6ac6412072f67cf767ebbde4133a5b2e88e76dc6187fa7104cd16f783399"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.105"
+version = "1.0.106"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac5d00fc561ba2724df6758a17de23df5914f20e41cb00f94d5b7ae42fffaff8"
+checksum = "9e549e3abf4fb8621bd1609f11dfc9f5e50320802273b12f3811a67e6716ea6c"
 dependencies = [
  "proc-macro2",
  "quote 1.0.3",
@@ -6939,9 +6938,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.50"
+version = "1.0.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78a7a12c167809363ec3bd7329fc0a3369056996de43c4b37ef3cd54a6ce4867"
+checksum = "da07b57ee2623368351e9a0488bb0b261322a15a6e0ae53e243cbdc0f4208da9"
 dependencies = [
  "itoa",
  "ryu",
@@ -7390,7 +7389,7 @@ dependencies = [
  "sp-storage",
  "substrate-bip39",
  "tiny-bip39",
- "tiny-keccak 2.0.1",
+ "tiny-keccak 2.0.2",
  "twox-hash",
  "wasmi",
  "zeroize",
@@ -7962,7 +7961,7 @@ dependencies = [
  "sc-rpc-api",
  "serde",
  "sp-storage",
- "tokio 0.2.13",
+ "tokio 0.2.16",
 ]
 
 [[package]]
@@ -7998,7 +7997,7 @@ dependencies = [
  "hyper 0.13.4",
  "log",
  "prometheus",
- "tokio 0.2.13",
+ "tokio 0.2.16",
 ]
 
 [[package]]
@@ -8400,9 +8399,9 @@ dependencies = [
 
 [[package]]
 name = "tiny-keccak"
-version = "2.0.1"
+version = "2.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2953ca5148619bc99695c1274cb54c5275bbb913c6adad87e72eaf8db9787f69"
+checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
 dependencies = [
  "crunchy",
 ]
@@ -8443,9 +8442,9 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "0.2.13"
+version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fa5e81d6bc4e67fe889d5783bd2a128ab2e0cfa487e0be16b6a8d177b101616"
+checksum = "ee5a0dd887e37d37390c13ff8ac830f992307fe30a1fff0ab8427af67211ba28"
 dependencies = [
  "bytes 0.5.4",
  "fnv",
@@ -8577,7 +8576,7 @@ checksum = "4adb8b3e5f86b707f1b54e7c15b6de52617a823608ccda98a15d3a24222f265a"
 dependencies = [
  "futures-core",
  "rustls 0.17.0",
- "tokio 0.2.13",
+ "tokio 0.2.16",
  "webpki",
 ]
 
@@ -8689,7 +8688,7 @@ dependencies = [
  "futures-sink",
  "log",
  "pin-project-lite",
- "tokio 0.2.13",
+ "tokio 0.2.16",
 ]
 
 [[package]]
@@ -9226,18 +9225,18 @@ dependencies = [
 
 [[package]]
 name = "wast"
-version = "12.0.0"
+version = "13.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0615ba420811bcda39cf80e8a1bd75997aec09222bda35165920a07ef15cc695"
+checksum = "5b20abd8b4a26f7e0d4dd5e357e90a3d555ec190e94472c9b2b27c5b9777f9ae"
 dependencies = [
  "leb128",
 ]
 
 [[package]]
 name = "wat"
-version = "1.0.13"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "095f615fbfcae695e3a4cea7d9f02f70561c81274c0142f45a12bf1e154d08bd"
+checksum = "51a615830ee3e7200b505c441fec09aac2f114deae69df52f215cb828ba112c4"
 dependencies = [
  "wast",
 ]

substrate/client/db/Cargo.toml

@@ -11,7 +11,6 @@ description = "Client backend that uses RocksDB database as storage."
 [dependencies]
 parking_lot = "0.10.0"
 log = "0.4.8"
-rand = "0.7"
 kvdb = "0.5.0"
 kvdb-rocksdb = { version = "0.7", optional = true }
 kvdb-memorydb = "0.5.0"

substrate/client/db/src/bench.rs

@@ -17,10 +17,8 @@
 //! State backend that's useful for benchmarking
 
 use std::sync::Arc;
-use std::path::PathBuf;
 use std::cell::{Cell, RefCell};
 use std::collections::HashMap;
-use rand::Rng;
 
 use hash_db::{Prefix, Hasher};
 use sp_trie::{MemoryDB, prefixed_key};
@@ -29,12 +27,14 @@ use sp_runtime::traits::{Block as BlockT, HashFor};
 use sp_runtime::Storage;
 use sp_state_machine::{DBValue, backend::Backend as StateBackend};
 use kvdb::{KeyValueDB, DBTransaction};
-use kvdb_rocksdb::{Database, DatabaseConfig};
+use crate::storage_cache::{CachingState, SharedCache, new_shared_cache};
 
 type DbState<B> = sp_state_machine::TrieBackend<
 	Arc<dyn sp_state_machine::Storage<HashFor<B>>>, HashFor<B>
 >;
 
+type State<B> = CachingState<DbState<B>, B>;
+
 struct StorageDb<Block: BlockT> {
 	db: Arc<dyn KeyValueDB>,
 	_block: std::marker::PhantomData<Block>,
@@ -50,37 +50,30 @@ impl<Block: BlockT> sp_state_machine::Storage<HashFor<Block>> for StorageDb<Bloc
 
 /// State that manages the backend database reference. Allows runtime to control the database.
 pub struct BenchmarkingState<B: BlockT> {
-	path: PathBuf,
 	root: Cell<B::Hash>,
 	genesis_root: B::Hash,
-	state: RefCell<Option<DbState<B>>>,
+	state: RefCell<Option<State<B>>>,
 	db: Cell<Option<Arc<dyn KeyValueDB>>>,
 	genesis: HashMap<Vec<u8>, (Vec<u8>, i32)>,
 	record: Cell<Vec<Vec<u8>>>,
-	cache_size_mb: Option<usize>,
+	shared_cache: SharedCache<B>, // shared cache is always empty
 }
 
 impl<B: BlockT> BenchmarkingState<B> {
 	/// Create a new instance that creates a database in a temporary dir.
-	pub fn new(genesis: Storage, cache_size_mb: Option<usize>) -> Result<Self, String> {
-		let temp_dir = PathBuf::from(std::env::temp_dir());
-		let name: String = rand::thread_rng().sample_iter(&rand::distributions::Alphanumeric).take(10).collect();
-		let path = temp_dir.join(&name);
-
+	pub fn new(genesis: Storage, _cache_size_mb: Option<usize>) -> Result<Self, String> {
 		let mut root = B::Hash::default();
 		let mut mdb = MemoryDB::<HashFor<B>>::default();
 		sp_state_machine::TrieDBMut::<HashFor<B>>::new(&mut mdb, &mut root);
 
-		std::fs::create_dir(&path).map_err(|_| String::from("Error creating temp dir"))?;
 		let mut state = BenchmarkingState {
 			state: RefCell::new(None),
 			db: Cell::new(None),
-			path,
 			root: Cell::new(root),
 			genesis: Default::default(),
 			genesis_root: Default::default(),
 			record: Default::default(),
-			cache_size_mb,
+			shared_cache: new_shared_cache(0, (1, 10)),
 		};
 
 		state.reopen()?;
@@ -96,41 +89,25 @@ impl<B: BlockT> BenchmarkingState<B> {
 		state.genesis = transaction.clone().drain();
 		state.genesis_root = root.clone();
 		state.commit(root, transaction)?;
+		state.record.take();
 		Ok(state)
 	}
 
 	fn reopen(&self) -> Result<(), String> {
 		*self.state.borrow_mut() = None;
-		self.db.set(None);
-		let mut db_config = DatabaseConfig::with_columns(1);
-		if let Some(size) = &self.cache_size_mb {
-			db_config.memory_budget.insert(0, *size);
-		}
-		let path = self.path.to_str()
-			.ok_or_else(|| String::from("Invalid database path"))?;
-		let db = Arc::new(Database::open(&db_config, &path).map_err(|e| format!("Error opening database: {:?}", e))?);
+		let db = match self.db.take() {
+			Some(db) => db,
+			None => Arc::new(::kvdb_memorydb::create(1)),
+		};
 		self.db.set(Some(db.clone()));
 		let storage_db = Arc::new(StorageDb::<B> { db, _block: Default::default() });
-		*self.state.borrow_mut() = Some(DbState::<B>::new(storage_db, self.root.get()));
+		*self.state.borrow_mut() = Some(State::new(
+			DbState::<B>::new(storage_db, self.root.get()),
+			self.shared_cache.clone(),
+			None
+		));
 		Ok(())
 	}
-
-	fn kill(&self) -> Result<(), String> {
-		self.db.set(None);
-		*self.state.borrow_mut() = None;
-		let mut root = B::Hash::default();
-		let mut mdb = MemoryDB::<HashFor<B>>::default();
-		sp_state_machine::TrieDBMut::<HashFor<B>>::new(&mut mdb, &mut root);
-		self.root.set(root);
-
-		std::fs::remove_dir_all(&self.path).map_err(|_| "Error removing database dir".into())
-	}
-}
-
-impl<B: BlockT> Drop for BenchmarkingState<B> {
-	fn drop(&mut self) {
-		self.kill().ok();
-	}
 }
 
 fn state_err() -> String {
@@ -278,6 +255,7 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
 			self.record.set(keys);
 			db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?;
 			self.root.set(storage_root);
+			self.db.set(Some(db))
 		} else {
 			return Err("Trying to commit to a closed db".into())
 		}
@@ -296,11 +274,9 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
 				}
 			}
 			db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?;
+			self.db.set(Some(db));
 		}
 
-		self.db.set(None);
-		*self.state.borrow_mut() = None;
-
 		self.root.set(self.genesis_root.clone());
 		self.reopen()?;
 		Ok(())
@@ -317,6 +293,6 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
 
 impl<Block: BlockT> std::fmt::Debug for BenchmarkingState<Block> {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		write!(f, "DB at {:?}", self.path)
+		write!(f, "Bench DB")
 	}
 }