i/o stats for backend databases (#4525)

substrate/client/db/src/lib.rs

@@ -39,7 +39,7 @@ use std::path::PathBuf;
 use std::io;
 use std::collections::{HashMap, HashSet};
 
-use sc_client_api::{execution_extensions::ExecutionExtensions, BadBlocks, ForkBlocks};
+use sc_client_api::{execution_extensions::ExecutionExtensions, ForkBlocks, UsageInfo, MemoryInfo, BadBlocks, IoInfo};
 use sc_client_api::backend::NewBlockState;
 use sc_client_api::backend::{StorageCollection, ChildStorageCollection};
 use sp_blockchain::{
@@ -831,7 +831,48 @@ where
 	}
 }
 
-/// Disk backend. Keeps data in a key-value store. In archive mode, trie nodes are kept from all blocks.
+/// Frozen `value` at time `at`.
+///
+/// Used as inner structure under lock in `FrozenForDuration`.
+struct Frozen<T: Clone> {
+	at: std::time::Instant,
+	value: T,
+}
+
+/// Some value frozen for period of time.
+///
+/// If time `duration` not passed since the value was instantiated,
+/// current frozen value is returned. Otherwise, you have to provide
+/// a new value which will be again frozen for `duration`.
+pub(crate) struct FrozenForDuration<T: Clone> {
+	duration: std::time::Duration,
+	value: RwLock<Frozen<T>>,
+}
+
+impl<T: Clone> FrozenForDuration<T> {
+	fn new(duration: std::time::Duration, initial: T) -> Self {
+		Self {
+			duration,
+			value: Frozen { at: std::time::Instant::now(), value: initial }.into(),
+		}
+	}
+
+	fn take_or_else<F>(&self, f: F) -> T where F: FnOnce() -> T {
+		if self.value.read().at.elapsed() > self.duration {
+			let mut write_lock = self.value.write();
+			let new_value = f();
+			write_lock.at = std::time::Instant::now();
+			write_lock.value = new_value.clone();
+			new_value
+		} else {
+			self.value.read().value.clone()
+		}
+	}
+}
+
+/// Disk backend.
+///
+/// Disk backend keps data in a key-value store. In archive mode, trie nodes are kept from all blocks.
 /// Otherwise, trie nodes are kept only from some recent blocks.
 pub struct Backend<Block: BlockT> {
 	storage: Arc<StorageDb<Block>>,
@@ -845,6 +886,7 @@ pub struct Backend<Block: BlockT> {
 	shared_cache: SharedCache<Block, Blake2Hasher>,
 	import_lock: RwLock<()>,
 	is_archive: bool,
+	io_stats: FrozenForDuration<kvdb::IoStats>,
 }
 
 impl<Block: BlockT<Hash=H256>> Backend<Block> {
@@ -906,6 +948,7 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 			),
 			import_lock: Default::default(),
 			is_archive: is_archive_pruning,
+			io_stats: FrozenForDuration::new(std::time::Duration::from_secs(1), kvdb::IoStats::empty()),
 		})
 	}
 
@@ -1492,9 +1535,31 @@ impl<Block> sc_client_api::backend::Backend<Block, Blake2Hasher> for Backend<Blo
 		Some(self.offchain_storage.clone())
 	}
 
+	fn usage_info(&self) -> Option<UsageInfo> {
+		let io_stats = self.io_stats.take_or_else(|| self.storage.db.io_stats(kvdb::IoStatsKind::SincePrevious));
+		let database_cache = parity_util_mem::malloc_size(&*self.storage.db);
+		let state_cache = (*&self.shared_cache).lock().used_storage_cache_size();
+
+		Some(UsageInfo {
+			memory: MemoryInfo {
+				state_cache,
+				database_cache,
+			},
+			io: IoInfo {
+				transactions: io_stats.transactions,
+				bytes_read: io_stats.bytes_read,
+				bytes_written: io_stats.bytes_written,
+				writes: io_stats.writes,
+				reads: io_stats.reads,
+				average_transaction_size: io_stats.avg_transaction_size() as u64,
+			},
+		})
+	}
+
 	fn revert(&self, n: NumberFor<Block>, revert_finalized: bool) -> ClientResult<NumberFor<Block>> {
 		let mut best_number = self.blockchain.info().best_number;
 		let mut best_hash = self.blockchain.info().best_hash;
+
 		let finalized = self.blockchain.info().finalized_number;
 
 		let revertible = best_number - finalized;
@@ -1563,11 +1628,6 @@ impl<Block> sc_client_api::backend::Backend<Block, Blake2Hasher> for Backend<Blo
 		&self.blockchain
 	}
 
-	fn used_state_cache_size(&self) -> Option<usize> {
-		let used = (*&self.shared_cache).lock().used_storage_cache_size();
-		Some(used)
-	}
-
 	fn state_at(&self, block: BlockId<Block>) -> ClientResult<Self::State> {
 		use sc_client::blockchain::HeaderBackend as BcHeaderBackend;
 

substrate/client/db/src/light.rs

@@ -22,7 +22,7 @@ use parking_lot::RwLock;
 
 use kvdb::{KeyValueDB, DBTransaction};
 
-use sc_client_api::backend::{AuxStore, NewBlockState};
+use sc_client_api::{backend::{AuxStore, NewBlockState}, UsageInfo};
 use sc_client::blockchain::{
 	BlockStatus, Cache as BlockchainCache,Info as BlockchainInfo,
 };
@@ -30,7 +30,7 @@ use sc_client::cht;
 use sp_blockchain::{
 	CachedHeaderMetadata, HeaderMetadata, HeaderMetadataCache,
 	Error as ClientError, Result as ClientResult,
-	HeaderBackend as BlockchainHeaderBackend, 
+	HeaderBackend as BlockchainHeaderBackend,
 	well_known_cache_keys,
 };
 use sc_client::light::blockchain::Storage as LightBlockchainStorage;
@@ -40,7 +40,7 @@ use sp_runtime::generic::{DigestItem, BlockId};
 use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, One, NumberFor};
 use crate::cache::{DbCacheSync, DbCache, ComplexBlockId, EntryType as CacheEntryType};
 use crate::utils::{self, meta_keys, Meta, db_err, read_db, block_id_to_lookup_key, read_meta};
-use crate::DatabaseSettings;
+use crate::{DatabaseSettings, FrozenForDuration};
 use log::{trace, warn, debug};
 
 pub(crate) mod columns {
@@ -64,6 +64,7 @@ pub struct LightStorage<Block: BlockT> {
 	meta: RwLock<Meta<NumberFor<Block>, Block::Hash>>,
 	cache: Arc<DbCacheSync<Block>>,
 	header_metadata_cache: HeaderMetadataCache<Block>,
+	io_stats: FrozenForDuration<kvdb::IoStats>,
 }
 
 impl<Block> LightStorage<Block>
@@ -102,6 +103,7 @@ impl<Block> LightStorage<Block>
 			meta: RwLock::new(meta),
 			cache: Arc::new(DbCacheSync(RwLock::new(cache))),
 			header_metadata_cache: HeaderMetadataCache::default(),
+			io_stats: FrozenForDuration::new(std::time::Duration::from_secs(1), kvdb::IoStats::empty()),
 		})
 	}
 
@@ -548,6 +550,28 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
 	fn cache(&self) -> Option<Arc<dyn BlockchainCache<Block>>> {
 		Some(self.cache.clone())
 	}
+
+	fn usage_info(&self) -> Option<UsageInfo> {
+		use sc_client_api::{MemoryInfo, IoInfo};
+
+		let database_cache = parity_util_mem::malloc_size(&*self.db);
+		let io_stats = self.io_stats.take_or_else(|| self.db.io_stats(kvdb::IoStatsKind::SincePrevious));
+
+		Some(UsageInfo {
+			memory: MemoryInfo {
+				database_cache,
+				state_cache: 0,
+			},
+			io: IoInfo {
+				transactions: io_stats.transactions,
+				bytes_read: io_stats.bytes_read,
+				bytes_written: io_stats.bytes_written,
+				writes: io_stats.writes,
+				reads: io_stats.reads,
+				average_transaction_size: io_stats.avg_transaction_size() as u64,
+			}
+		})
+	}
 }
 
 /// Build the key for inserting header-CHT at given block.