i/o stats for backend databases (#4525)

substrate/client/Cargo.toml

@@ -18,7 +18,7 @@ hash-db = { version = "0.15.2" }
 hex-literal = { version = "0.2.1" }
 sp-inherents = { version = "2.0.0", path = "../primitives/inherents" }
 sp-keyring = { version = "2.0.0", path = "../primitives/keyring" }
-kvdb = "0.2.0"
+kvdb = "0.3.0"
 log = { version = "0.4.8" }
 parking_lot = { version = "0.9.0" }
 sp-core = { version = "2.0.0", path = "../primitives/core" }
@@ -36,5 +36,5 @@ tracing = "0.1.10"
 env_logger = "0.7.0"
 tempfile = "3.1.0"
 substrate-test-runtime-client = { version = "2.0.0", path = "../test-utils/runtime/client" }
-kvdb-memorydb = "0.2.0"
+kvdb-memorydb = "0.3.0"
 sp-panic-handler = { version = "2.0.0", path = "../primitives/panic-handler" }

substrate/client/api/Cargo.toml

@@ -17,7 +17,7 @@ sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" }
 hex-literal = { version = "0.2.1" }
 sp-inherents = { version = "2.0.0", default-features = false, path = "../../primitives/inherents" }
 sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" }
-kvdb = "0.2.0"
+kvdb = "0.3.0"
 log = { version = "0.4.8" }
 parking_lot = { version = "0.9.0" }
 sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" }

substrate/client/api/src/backend.rs

@@ -29,6 +29,7 @@ use crate::{
 		Backend as BlockchainBackend, well_known_cache_keys
 	},
 	light::RemoteBlockchain,
+	UsageInfo,
 };
 use sp_blockchain;
 use sp_consensus::BlockOrigin;
@@ -261,8 +262,8 @@ pub trait Backend<Block, H>: AuxStore + Send + Sync where
 	/// Returns reference to blockchain backend.
 	fn blockchain(&self) -> &Self::Blockchain;
 
-	/// Returns the used state cache, if existent.
-	fn used_state_cache_size(&self) -> Option<usize>;
+	/// Returns current usage statistics.
+	fn usage_info(&self) -> Option<UsageInfo>;
 
 	/// Returns reference to changes trie storage.
 	fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage>;

substrate/client/api/src/client.rs

@@ -16,7 +16,7 @@
 
 //! A set of APIs supported by the client along with their primitives.
 
-use std::collections::HashSet;
+use std::{fmt, collections::HashSet};
 use futures::channel::mpsc;
 use sp_core::storage::StorageKey;
 use sp_runtime::{
@@ -93,8 +93,61 @@ pub trait ProvideUncles<Block: BlockT> {
 pub struct ClientInfo<Block: BlockT> {
 	/// Best block hash.
 	pub chain: Info<Block>,
-	/// State Cache Size currently used by the backend
-	pub used_state_cache_size: Option<usize>,
+	/// Usage info, if backend supports this.
+	pub usage: Option<UsageInfo>,
+}
+
+/// Memory statistics for client instance.
+#[derive(Default, Clone, Debug)]
+pub struct MemoryInfo {
+	/// Size of state cache.
+	pub state_cache: usize,
+	/// Size of backend database cache.
+	pub database_cache: usize,
+}
+
+/// I/O statistics for client instance.
+#[derive(Default, Clone, Debug)]
+pub struct IoInfo {
+	/// Number of transactions.
+	pub transactions: u64,
+	/// Total bytes read from disk.
+	pub bytes_read: u64,
+	/// Total bytes written to disk.
+	pub bytes_written: u64,
+	/// Total key writes to disk.
+	pub writes: u64,
+	/// Total key reads from disk.
+	pub reads: u64,
+	/// Average size of the transaction.
+	pub average_transaction_size: u64,
+}
+
+/// Usage statistics for running client instance.
+///
+/// Returning backend determines the scope of these stats,
+/// but usually it is either from service start or from previous
+/// gathering of the statistics.
+#[derive(Default, Clone, Debug)]
+pub struct UsageInfo {
+	/// Memory statistics.
+	pub memory: MemoryInfo,
+	/// I/O statistics.
+	pub io: IoInfo,
+}
+
+impl fmt::Display for UsageInfo {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		write!(f,
+			"caches: ({} state, {} db overlay), i/o: ({} tx, {} write, {} read, {} tx size)",
+			self.memory.state_cache,
+			self.memory.database_cache,
+			self.io.transactions,
+			self.io.bytes_written,
+			self.io.bytes_read,
+			self.io.average_transaction_size,
+		)
+	}
 }
 
 /// Summary of an imported block

substrate/client/api/src/light.rs

@@ -32,7 +32,8 @@ use sp_blockchain::{
 	HeaderMetadata, well_known_cache_keys, HeaderBackend, Cache as BlockchainCache,
 	Error as ClientError, Result as ClientResult,
 };
-use crate::backend::{ AuxStore, NewBlockState };
+use crate::{backend::{AuxStore, NewBlockState}, UsageInfo};
+
 /// Remote call request.
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub struct RemoteCallRequest<Header: HeaderT> {
@@ -274,6 +275,9 @@ pub trait Storage<Block: BlockT>: AuxStore + HeaderBackend<Block> + HeaderMetada
 
 	/// Get storage cache.
 	fn cache(&self) -> Option<Arc<dyn BlockchainCache<Block>>>;
+
+	/// Get storage usage statistics.
+	fn usage_info(&self) -> Option<UsageInfo>;
 }
 
 /// Remote header.

substrate/client/cli/src/informant.rs

@@ -18,7 +18,7 @@
 
 use sc_client_api::BlockchainEvents;
 use futures::{StreamExt, TryStreamExt, FutureExt, future, compat::Stream01CompatExt};
-use log::{info, warn};
+use log::{info, warn, trace};
 use sp_runtime::traits::Header;
 use sc_service::AbstractService;
 use std::time::Duration;
@@ -36,6 +36,11 @@ pub fn build(service: &impl AbstractService) -> impl futures::Future<Output = ()
 		.compat()
 		.try_for_each(move |(net_status, _)| {
 			let info = client.usage_info();
+			if let Some(ref usage) = info.usage {
+				trace!(target: "usage", "Usage statistics: {}", usage);
+			} else {
+				trace!(target: "usage", "Usage statistics not displayed as backend does not provide it")
+			}
 			display.display(&info, net_status);
 			future::ok(())
 		});

substrate/client/db/Cargo.toml

@@ -7,17 +7,19 @@ edition = "2018"
 [dependencies]
 parking_lot = "0.9.0"
 log = "0.4.8"
-kvdb = "0.2.0"
-kvdb-rocksdb = { version = "0.3", optional = true }
-kvdb-memorydb = "0.2.0"
+kvdb = "0.3.0"
+kvdb-rocksdb = { version = "0.4", optional = true }
+kvdb-memorydb = "0.3.0"
 linked-hash-map = "0.5.2"
 hash-db = "0.15.2"
+parity-util-mem = { version = "0.4", default-features = false, features = ["std"] }
+codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] }
+
 sc-client-api = { version = "2.0.0", path = "../api" }
 sp-core = { version = "2.0.0", path = "../../primitives/core" }
 sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" }
 sc-client = { version = "2.0.0", path = "../" }
 sp-state-machine = { version = "2.0.0", path = "../../primitives/state-machine" }
-codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] }
 sc-executor = { version = "2.0.0", path = "../executor" }
 sc-state-db = { version = "2.0.0", path = "../state-db" }
 sp-trie = { version = "2.0.0", path = "../../primitives/trie" }

substrate/client/db/src/lib.rs

@@ -39,7 +39,7 @@ use std::path::PathBuf;
 use std::io;
 use std::collections::{HashMap, HashSet};
 
-use sc_client_api::{execution_extensions::ExecutionExtensions, BadBlocks, ForkBlocks};
+use sc_client_api::{execution_extensions::ExecutionExtensions, ForkBlocks, UsageInfo, MemoryInfo, BadBlocks, IoInfo};
 use sc_client_api::backend::NewBlockState;
 use sc_client_api::backend::{StorageCollection, ChildStorageCollection};
 use sp_blockchain::{
@@ -831,7 +831,48 @@ where
 	}
 }
 
-/// Disk backend. Keeps data in a key-value store. In archive mode, trie nodes are kept from all blocks.
+/// Frozen `value` at time `at`.
+///
+/// Used as inner structure under lock in `FrozenForDuration`.
+struct Frozen<T: Clone> {
+	at: std::time::Instant,
+	value: T,
+}
+
+/// Some value frozen for period of time.
+///
+/// If time `duration` not passed since the value was instantiated,
+/// current frozen value is returned. Otherwise, you have to provide
+/// a new value which will be again frozen for `duration`.
+pub(crate) struct FrozenForDuration<T: Clone> {
+	duration: std::time::Duration,
+	value: RwLock<Frozen<T>>,
+}
+
+impl<T: Clone> FrozenForDuration<T> {
+	fn new(duration: std::time::Duration, initial: T) -> Self {
+		Self {
+			duration,
+			value: Frozen { at: std::time::Instant::now(), value: initial }.into(),
+		}
+	}
+
+	fn take_or_else<F>(&self, f: F) -> T where F: FnOnce() -> T {
+		if self.value.read().at.elapsed() > self.duration {
+			let mut write_lock = self.value.write();
+			let new_value = f();
+			write_lock.at = std::time::Instant::now();
+			write_lock.value = new_value.clone();
+			new_value
+		} else {
+			self.value.read().value.clone()
+		}
+	}
+}
+
+/// Disk backend.
+///
+/// Disk backend keps data in a key-value store. In archive mode, trie nodes are kept from all blocks.
 /// Otherwise, trie nodes are kept only from some recent blocks.
 pub struct Backend<Block: BlockT> {
 	storage: Arc<StorageDb<Block>>,
@@ -845,6 +886,7 @@ pub struct Backend<Block: BlockT> {
 	shared_cache: SharedCache<Block, Blake2Hasher>,
 	import_lock: RwLock<()>,
 	is_archive: bool,
+	io_stats: FrozenForDuration<kvdb::IoStats>,
 }
 
 impl<Block: BlockT<Hash=H256>> Backend<Block> {
@@ -906,6 +948,7 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 			),
 			import_lock: Default::default(),
 			is_archive: is_archive_pruning,
+			io_stats: FrozenForDuration::new(std::time::Duration::from_secs(1), kvdb::IoStats::empty()),
 		})
 	}
 
@@ -1492,9 +1535,31 @@ impl<Block> sc_client_api::backend::Backend<Block, Blake2Hasher> for Backend<Blo
 		Some(self.offchain_storage.clone())
 	}
 
+	fn usage_info(&self) -> Option<UsageInfo> {
+		let io_stats = self.io_stats.take_or_else(|| self.storage.db.io_stats(kvdb::IoStatsKind::SincePrevious));
+		let database_cache = parity_util_mem::malloc_size(&*self.storage.db);
+		let state_cache = (*&self.shared_cache).lock().used_storage_cache_size();
+
+		Some(UsageInfo {
+			memory: MemoryInfo {
+				state_cache,
+				database_cache,
+			},
+			io: IoInfo {
+				transactions: io_stats.transactions,
+				bytes_read: io_stats.bytes_read,
+				bytes_written: io_stats.bytes_written,
+				writes: io_stats.writes,
+				reads: io_stats.reads,
+				average_transaction_size: io_stats.avg_transaction_size() as u64,
+			},
+		})
+	}
+
 	fn revert(&self, n: NumberFor<Block>, revert_finalized: bool) -> ClientResult<NumberFor<Block>> {
 		let mut best_number = self.blockchain.info().best_number;
 		let mut best_hash = self.blockchain.info().best_hash;
+
 		let finalized = self.blockchain.info().finalized_number;
 
 		let revertible = best_number - finalized;
@@ -1563,11 +1628,6 @@ impl<Block> sc_client_api::backend::Backend<Block, Blake2Hasher> for Backend<Blo
 		&self.blockchain
 	}
 
-	fn used_state_cache_size(&self) -> Option<usize> {
-		let used = (*&self.shared_cache).lock().used_storage_cache_size();
-		Some(used)
-	}
-
 	fn state_at(&self, block: BlockId<Block>) -> ClientResult<Self::State> {
 		use sc_client::blockchain::HeaderBackend as BcHeaderBackend;
 

substrate/client/db/src/light.rs

@@ -22,7 +22,7 @@ use parking_lot::RwLock;
 
 use kvdb::{KeyValueDB, DBTransaction};
 
-use sc_client_api::backend::{AuxStore, NewBlockState};
+use sc_client_api::{backend::{AuxStore, NewBlockState}, UsageInfo};
 use sc_client::blockchain::{
 	BlockStatus, Cache as BlockchainCache,Info as BlockchainInfo,
 };
@@ -30,7 +30,7 @@ use sc_client::cht;
 use sp_blockchain::{
 	CachedHeaderMetadata, HeaderMetadata, HeaderMetadataCache,
 	Error as ClientError, Result as ClientResult,
-	HeaderBackend as BlockchainHeaderBackend, 
+	HeaderBackend as BlockchainHeaderBackend,
 	well_known_cache_keys,
 };
 use sc_client::light::blockchain::Storage as LightBlockchainStorage;
@@ -40,7 +40,7 @@ use sp_runtime::generic::{DigestItem, BlockId};
 use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, One, NumberFor};
 use crate::cache::{DbCacheSync, DbCache, ComplexBlockId, EntryType as CacheEntryType};
 use crate::utils::{self, meta_keys, Meta, db_err, read_db, block_id_to_lookup_key, read_meta};
-use crate::DatabaseSettings;
+use crate::{DatabaseSettings, FrozenForDuration};
 use log::{trace, warn, debug};
 
 pub(crate) mod columns {
@@ -64,6 +64,7 @@ pub struct LightStorage<Block: BlockT> {
 	meta: RwLock<Meta<NumberFor<Block>, Block::Hash>>,
 	cache: Arc<DbCacheSync<Block>>,
 	header_metadata_cache: HeaderMetadataCache<Block>,
+	io_stats: FrozenForDuration<kvdb::IoStats>,
 }
 
 impl<Block> LightStorage<Block>
@@ -102,6 +103,7 @@ impl<Block> LightStorage<Block>
 			meta: RwLock::new(meta),
 			cache: Arc::new(DbCacheSync(RwLock::new(cache))),
 			header_metadata_cache: HeaderMetadataCache::default(),
+			io_stats: FrozenForDuration::new(std::time::Duration::from_secs(1), kvdb::IoStats::empty()),
 		})
 	}
 
@@ -548,6 +550,28 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
 	fn cache(&self) -> Option<Arc<dyn BlockchainCache<Block>>> {
 		Some(self.cache.clone())
 	}
+
+	fn usage_info(&self) -> Option<UsageInfo> {
+		use sc_client_api::{MemoryInfo, IoInfo};
+
+		let database_cache = parity_util_mem::malloc_size(&*self.db);
+		let io_stats = self.io_stats.take_or_else(|| self.db.io_stats(kvdb::IoStatsKind::SincePrevious));
+
+		Some(UsageInfo {
+			memory: MemoryInfo {
+				database_cache,
+				state_cache: 0,
+			},
+			io: IoInfo {
+				transactions: io_stats.transactions,
+				bytes_read: io_stats.bytes_read,
+				bytes_written: io_stats.bytes_written,
+				writes: io_stats.writes,
+				reads: io_stats.reads,
+				average_transaction_size: io_stats.avg_transaction_size() as u64,
+			}
+		})
+	}
 }
 
 /// Build the key for inserting header-CHT at given block.

substrate/client/service/src/builder.rs

@@ -932,11 +932,6 @@ ServiceBuilder<
 			let bandwidth_download = net_status.average_download_per_sec;
 			let bandwidth_upload = net_status.average_upload_per_sec;
 
-			let used_state_cache_size = match info.used_state_cache_size {
-				Some(size) => size,
-				None => 0,
-			};
-
 			// get cpu usage and memory usage of this process
 			let (cpu_usage, memory) = if let Some(self_pid) = self_pid {
 				if sys.refresh_process(self_pid) {
@@ -959,7 +954,10 @@ ServiceBuilder<
 				"finalized_hash" => ?info.chain.finalized_hash,
 				"bandwidth_download" => bandwidth_download,
 				"bandwidth_upload" => bandwidth_upload,
-				"used_state_cache_size" => used_state_cache_size,
+				"used_state_cache_size" => info.usage.as_ref().map(|usage| usage.memory.state_cache).unwrap_or(0),
+				"used_db_cache_size" => info.usage.as_ref().map(|usage| usage.memory.database_cache).unwrap_or(0),
+				"disk_read_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_read).unwrap_or(0),
+				"disk_write_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_written).unwrap_or(0),
 			);
 			let _ = record_metrics!(
 				"peers" => num_peers,
@@ -970,7 +968,10 @@ ServiceBuilder<
 				"finalized_height" => finalized_number,
 				"bandwidth_download" => bandwidth_download,
 				"bandwidth_upload" => bandwidth_upload,
-				"used_state_cache_size" => used_state_cache_size,
+				"used_state_cache_size" => info.usage.as_ref().map(|usage| usage.memory.state_cache).unwrap_or(0),
+				"used_db_cache_size" => info.usage.as_ref().map(|usage| usage.memory.database_cache).unwrap_or(0),
+				"disk_read_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_read).unwrap_or(0),
+				"disk_write_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_written).unwrap_or(0),
 			);
 
 			Ok(())

substrate/client/src/client.rs

@@ -1176,7 +1176,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 	pub fn usage_info(&self) -> ClientInfo<Block> {
 		ClientInfo {
 			chain: self.chain_info(),
-			used_state_cache_size: self.backend.used_state_cache_size(),
+			usage: self.backend.usage_info(),
 		}
 	}
 

substrate/client/src/in_mem.rs

@@ -37,6 +37,7 @@ use sc_client_api::{
 	blockchain::{
 		self, BlockStatus, HeaderBackend, well_known_cache_keys::Id as CacheKeyId
 	},
+	UsageInfo,
 };
 use crate::leaves::LeafSet;
 
@@ -449,6 +450,10 @@ impl<Block: BlockT> sc_client_api::light::Storage<Block> for Blockchain<Block>
 	fn cache(&self) -> Option<Arc<dyn blockchain::Cache<Block>>> {
 		None
 	}
+
+	fn usage_info(&self) -> Option<sc_client_api::UsageInfo> {
+		None
+	}
 }
 
 /// In-memory operation.
@@ -681,7 +686,7 @@ where
 		&self.blockchain
 	}
 
-	fn used_state_cache_size(&self) -> Option<usize> {
+	fn usage_info(&self) -> Option<UsageInfo> {
 		None
 	}
 

substrate/client/src/light/backend.rs

@@ -39,6 +39,7 @@ use sc_client_api::{
 		HeaderBackend as BlockchainHeaderBackend, well_known_cache_keys,
 	},
 	light::Storage as BlockchainStorage,
+	UsageInfo,
 };
 use crate::light::blockchain::Blockchain;
 use hash_db::Hasher;
@@ -186,8 +187,8 @@ impl<S, Block, H> ClientBackend<Block, H> for Backend<S, H> where
 		&self.blockchain
 	}
 
-	fn used_state_cache_size(&self) -> Option<usize> {
-		None
+	fn usage_info(&self) -> Option<UsageInfo> {
+		self.blockchain.storage().usage_info()
 	}
 
 	fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage> {

substrate/client/src/light/blockchain.rs

@@ -314,5 +314,9 @@ pub mod tests {
 		fn cache(&self) -> Option<Arc<dyn BlockchainCache<Block>>> {
 			None
 		}
+
+		fn usage_info(&self) -> Option<sc_client_api::UsageInfo> {
+			None
+		}
 	}
 }