Add DB Read/Write Tracking to Benchmarking Pipeline (#6386)

* initial mockup

* add and wipe

* track writes

* start to add to pipeline

* return all reads/writes

* Log reads and writes from bench db

* causes panic

* Allow multiple commits

* commit before ending benchmark

* doesn't work???

* fix

* Update lib.rs

* switch to struct for `BenchmarkResults`

* add to output

* fix test

* line width

* @kianenigma review

* Add Whitelist to DB Tracking in Benchmarks Pipeline (#6405)

* hardcoded whitelist

* Add whitelist to pipeline

* Remove whitelist pipeline from CLI, add to runtime

* clean-up unused db initialized whitelist

* Add regression analysis to DB Tracking (#6475)

* Add selector

* add tests

* debug formatter for easy formula

* Update client/db/src/bench.rs

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

Co-authored-by: arkpar <arkady.paronyan@gmail.com>
Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

substrate/primitives/externalities/src/lib.rs

@@ -233,6 +233,27 @@ pub trait Externalities: ExtensionStore {
 	///
 	/// Commits all changes to the database and clears all caches.
 	fn commit(&mut self);
+
+	/// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+	/// Benchmarking related functionality and shouldn't be used anywhere else!
+	/// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+	///
+	/// Gets the current read/write count for the benchmarking process.
+	fn read_write_count(&self) -> (u32, u32, u32, u32);
+
+	/// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+	/// Benchmarking related functionality and shouldn't be used anywhere else!
+	/// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+	///
+	/// Resets read/write count for the benchmarking process.
+	fn reset_read_write_count(&mut self);
+
+	/// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+	/// Benchmarking related functionality and shouldn't be used anywhere else!
+	/// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+	///
+	/// Adds new storage keys to the DB tracking whitelist.
+	fn set_whitelist(&mut self, new: Vec<Vec<u8>>);
 }
 
 /// Extension for the [`Externalities`] trait.

substrate/primitives/runtime-interface/src/impls.rs

@@ -365,6 +365,10 @@ impl<T: codec::Codec> PassBy for Option<T> {
 	type PassBy = Codec<Self>;
 }
 
+impl PassBy for (u32, u32, u32, u32) {
+	type PassBy = Codec<Self>;
+}
+
 /// Implement `PassBy` with `Inner` for the given fixed sized hash types.
 macro_rules! for_primitive_types {
 	{ $( $hash:ident $n:expr ),* $(,)? } => {

substrate/primitives/state-machine/src/backend.rs

@@ -212,7 +212,22 @@ pub trait Backend<H: Hasher>: std::fmt::Debug {
 	}
 
 	/// Commit given transaction to storage.
-	fn commit(&self, _: H::Out, _: Self::Transaction) -> Result<(), Self::Error> {
+	fn commit(&self, _: H::Out, _: Self::Transaction, _: StorageCollection) -> Result<(), Self::Error> {
+		unimplemented!()
+	}
+
+	/// Get the read/write count of the db
+	fn read_write_count(&self) -> (u32, u32, u32, u32) {
+		unimplemented!()
+	}
+
+	/// Get the read/write count of the db
+	fn reset_read_write_count(&self) {
+		unimplemented!()
+	}
+
+	/// Update the whitelist for tracking db reads/writes
+	fn set_whitelist(&self, _: Vec<Vec<u8>>) {
 		unimplemented!()
 	}
 }

substrate/primitives/state-machine/src/basic.rs

@@ -322,6 +322,18 @@ impl Externalities for BasicExternalities {
 	fn wipe(&mut self) {}
 
 	fn commit(&mut self) {}
+
+	fn read_write_count(&self) -> (u32, u32, u32, u32) {
+		unimplemented!("read_write_count is not supported in Basic")
+	}
+
+	fn reset_read_write_count(&mut self) {
+		unimplemented!("reset_read_write_count is not supported in Basic")
+	}
+
+	fn set_whitelist(&mut self, _: Vec<Vec<u8>>) {
+		unimplemented!("set_whitelist is not supported in Basic")
+	}
 }
 
 impl sp_externalities::ExtensionStore for BasicExternalities {

substrate/primitives/state-machine/src/ext.rs

@@ -590,9 +590,22 @@ where
 		self.backend.commit(
 			changes.transaction_storage_root,
 			changes.transaction,
+			changes.main_storage_changes,
 		).expect(EXT_NOT_ALLOWED_TO_FAIL);
 		self.mark_dirty();
 	}
+
+	fn read_write_count(&self) -> (u32, u32, u32, u32) {
+		self.backend.read_write_count()
+	}
+
+	fn reset_read_write_count(&mut self) {
+		self.backend.reset_read_write_count()
+	}
+
+	fn set_whitelist(&mut self, new: Vec<Vec<u8>>) {
+		self.backend.set_whitelist(new)
+	}
 }
 
 

substrate/primitives/state-machine/src/read_only.rs

@@ -185,6 +185,18 @@ impl<'a, H: Hasher, B: 'a + Backend<H>> Externalities for ReadOnlyExternalities<
 	fn wipe(&mut self) {}
 
 	fn commit(&mut self) {}
+
+	fn read_write_count(&self) -> (u32, u32, u32, u32) {
+		unimplemented!("read_write_count is not supported in ReadOnlyExternalities")
+	}
+
+	fn reset_read_write_count(&mut self) {
+		unimplemented!("reset_read_write_count is not supported in ReadOnlyExternalities")
+	}
+
+	fn set_whitelist(&mut self, _: Vec<Vec<u8>>) {
+		unimplemented!("set_whitelist is not supported in ReadOnlyExternalities")
+	}
 }
 
 impl<'a, H: Hasher, B: 'a + Backend<H>> sp_externalities::ExtensionStore for ReadOnlyExternalities<'a, H, B> {