diff --git a/substrate/Cargo.lock b/substrate/Cargo.lock index c5a097ff79..f0f096a659 100644 --- a/substrate/Cargo.lock +++ b/substrate/Cargo.lock @@ -368,7 +368,7 @@ dependencies = [ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "js-sys 0.3.34 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb-web 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-web 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "libp2p 0.13.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2249,33 +2249,36 @@ dependencies = [ [[package]] name = "kvdb" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "parity-bytes 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "kvdb-memorydb" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "kvdb 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "kvdb-rocksdb" -version = "0.3.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "fs-swap 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "interleaved-ordered 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)", "owning_ref 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "rocksdb 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2284,14 +2287,15 @@ dependencies = [ [[package]] name = "kvdb-web" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "js-sys 0.3.34 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb-memorydb 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-memorydb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "send_wrapper 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "wasm-bindgen 0.2.57 (registry+https://github.com/rust-lang/crates.io-index)", "web-sys 0.3.34 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4166,6 +4170,29 @@ dependencies = [ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "parity-util-mem" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem-derive 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "parity-util-mem-derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", + "synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "parity-wasm" version = "0.32.0" @@ -5070,8 +5097,8 @@ dependencies = [ "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb-memorydb 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-memorydb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -5106,7 +5133,7 @@ dependencies = [ "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -5134,12 +5161,13 @@ version = "2.0.0" dependencies = [ "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb-memorydb 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb-rocksdb 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-memorydb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-rocksdb 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "quickcheck 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "sc-client 2.0.0", @@ -8398,10 +8426,10 @@ dependencies = [ "checksum keccak-hasher 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3468207deea1359a0e921591ae9b4c928733d94eb9d6a2eeda994cfd59f42cf8" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum kv-log-macro 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c54d9f465d530a752e6ebdc217e081a7a614b48cb200f6f0aee21ba6bc9aabb" -"checksum kvdb 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cecee8d85a74f6b8284710d52a7d1196f09e31f8217e1f184a475b509d360554" -"checksum kvdb-memorydb 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0a5d70712b1fe0f02ce7ee36a962fcb0b15d0fe11262ba21a4aa839ef22cf60d" -"checksum kvdb-rocksdb 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "54cc6b52f7e511de9f07fd77cda70247adfc6b8192e4b5a1b6dbca416dc425b5" -"checksum kvdb-web 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "90cff04dc606356c88e3fbbf8033040b072cb57d03d6fed8b6d6372587d5931c" +"checksum kvdb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8396be0e5561ccd1bf7ff0b2007487cdd7a87a056873fe6ea906b35d4dbf7ed8" +"checksum kvdb-memorydb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4d25ef14155e418515c4839e9144c839de3506e68946f255a32b7f166095493d" +"checksum kvdb-rocksdb 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "af488cc16c3801705c8d681c3a32c8faa8fafc7fb5309dee0f573f3c6a19d395" +"checksum kvdb-web 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "37a0e36637fb86454de401e7cb88f40eb0ad1b9bcee837d46785e7c451f1ebf4" "checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" "checksum lazycell 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" @@ -8488,6 +8516,8 @@ dependencies = [ "checksum parity-scale-codec-derive 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "34e513ff3e406f3ede6796dcdc83d0b32ffb86668cea1ccf7363118abeb00476" "checksum parity-send-wrapper 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aa9777aa91b8ad9dd5aaa04a9b6bcb02c7f1deb952fca5a66034d5e63afc5c6f" "checksum parity-util-mem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8174d85e62c4d615fddd1ef67966bdc5757528891d0742f15b131ad04667b3f9" +"checksum parity-util-mem 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "900dd84654b048e5bad420bb341658fc2c4d7fea628c22bcf4621733e54859b4" +"checksum parity-util-mem-derive 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f557c32c6d268a07c921471619c0295f5efad3a0e76d4f97a05c091a51d110b2" "checksum parity-wasm 0.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "16ad52817c4d343339b3bc2e26861bd21478eda0b7509acf83505727000512ac" "checksum parity-wasm 0.41.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865" "checksum parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ab41b4aed082705d1056416ae4468b6ea99d52599ecf3169b00088d43113e337" diff --git a/substrate/client/Cargo.toml b/substrate/client/Cargo.toml index 2932eff21d..eaddd9f3dc 100644 --- a/substrate/client/Cargo.toml +++ b/substrate/client/Cargo.toml @@ -18,7 +18,7 @@ hash-db = { version = "0.15.2" } hex-literal = { version = "0.2.1" } sp-inherents = { version = "2.0.0", path = "../primitives/inherents" } sp-keyring = { version = "2.0.0", path = "../primitives/keyring" } -kvdb = "0.2.0" +kvdb = "0.3.0" log = { version = "0.4.8" } parking_lot = { version = "0.9.0" } sp-core = { version = "2.0.0", path = "../primitives/core" } @@ -36,5 +36,5 @@ tracing = "0.1.10" env_logger = "0.7.0" tempfile = "3.1.0" substrate-test-runtime-client = { version = "2.0.0", path = "../test-utils/runtime/client" } -kvdb-memorydb = "0.2.0" +kvdb-memorydb = "0.3.0" sp-panic-handler = { version = "2.0.0", path = "../primitives/panic-handler" } diff --git a/substrate/client/api/Cargo.toml b/substrate/client/api/Cargo.toml index 9e2e35e3a2..36360a6fb8 100644 --- a/substrate/client/api/Cargo.toml +++ b/substrate/client/api/Cargo.toml @@ -17,7 +17,7 @@ sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } hex-literal = { version = "0.2.1" } sp-inherents = { version = "2.0.0", default-features = false, path = "../../primitives/inherents" } sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } -kvdb = "0.2.0" +kvdb = "0.3.0" log = { version = "0.4.8" } parking_lot = { version = "0.9.0" } sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } diff --git a/substrate/client/api/src/backend.rs b/substrate/client/api/src/backend.rs index 90a0c9aa7b..a956b9eb22 100644 --- a/substrate/client/api/src/backend.rs +++ b/substrate/client/api/src/backend.rs @@ -29,6 +29,7 @@ use crate::{ Backend as BlockchainBackend, well_known_cache_keys }, light::RemoteBlockchain, + UsageInfo, }; use sp_blockchain; use sp_consensus::BlockOrigin; @@ -261,8 +262,8 @@ pub trait Backend: AuxStore + Send + Sync where /// Returns reference to blockchain backend. fn blockchain(&self) -> &Self::Blockchain; - /// Returns the used state cache, if existent. - fn used_state_cache_size(&self) -> Option; + /// Returns current usage statistics. + fn usage_info(&self) -> Option; /// Returns reference to changes trie storage. fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage>; diff --git a/substrate/client/api/src/client.rs b/substrate/client/api/src/client.rs index 9e095a897f..78a70bd5ea 100644 --- a/substrate/client/api/src/client.rs +++ b/substrate/client/api/src/client.rs @@ -16,7 +16,7 @@ //! A set of APIs supported by the client along with their primitives. -use std::collections::HashSet; +use std::{fmt, collections::HashSet}; use futures::channel::mpsc; use sp_core::storage::StorageKey; use sp_runtime::{ @@ -93,8 +93,61 @@ pub trait ProvideUncles { pub struct ClientInfo { /// Best block hash. pub chain: Info, - /// State Cache Size currently used by the backend - pub used_state_cache_size: Option, + /// Usage info, if backend supports this. + pub usage: Option, +} + +/// Memory statistics for client instance. +#[derive(Default, Clone, Debug)] +pub struct MemoryInfo { + /// Size of state cache. + pub state_cache: usize, + /// Size of backend database cache. + pub database_cache: usize, +} + +/// I/O statistics for client instance. +#[derive(Default, Clone, Debug)] +pub struct IoInfo { + /// Number of transactions. + pub transactions: u64, + /// Total bytes read from disk. + pub bytes_read: u64, + /// Total bytes written to disk. + pub bytes_written: u64, + /// Total key writes to disk. + pub writes: u64, + /// Total key reads from disk. + pub reads: u64, + /// Average size of the transaction. + pub average_transaction_size: u64, +} + +/// Usage statistics for running client instance. +/// +/// Returning backend determines the scope of these stats, +/// but usually it is either from service start or from previous +/// gathering of the statistics. +#[derive(Default, Clone, Debug)] +pub struct UsageInfo { + /// Memory statistics. + pub memory: MemoryInfo, + /// I/O statistics. + pub io: IoInfo, +} + +impl fmt::Display for UsageInfo { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, + "caches: ({} state, {} db overlay), i/o: ({} tx, {} write, {} read, {} tx size)", + self.memory.state_cache, + self.memory.database_cache, + self.io.transactions, + self.io.bytes_written, + self.io.bytes_read, + self.io.average_transaction_size, + ) + } } /// Summary of an imported block diff --git a/substrate/client/api/src/light.rs b/substrate/client/api/src/light.rs index fcdba1f627..b3e2e686ea 100644 --- a/substrate/client/api/src/light.rs +++ b/substrate/client/api/src/light.rs @@ -32,7 +32,8 @@ use sp_blockchain::{ HeaderMetadata, well_known_cache_keys, HeaderBackend, Cache as BlockchainCache, Error as ClientError, Result as ClientResult, }; -use crate::backend::{ AuxStore, NewBlockState }; +use crate::{backend::{AuxStore, NewBlockState}, UsageInfo}; + /// Remote call request. #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct RemoteCallRequest { @@ -274,6 +275,9 @@ pub trait Storage: AuxStore + HeaderBackend + HeaderMetada /// Get storage cache. fn cache(&self) -> Option>>; + + /// Get storage usage statistics. + fn usage_info(&self) -> Option; } /// Remote header. diff --git a/substrate/client/cli/src/informant.rs b/substrate/client/cli/src/informant.rs index 05111cb646..de7c376f09 100644 --- a/substrate/client/cli/src/informant.rs +++ b/substrate/client/cli/src/informant.rs @@ -18,7 +18,7 @@ use sc_client_api::BlockchainEvents; use futures::{StreamExt, TryStreamExt, FutureExt, future, compat::Stream01CompatExt}; -use log::{info, warn}; +use log::{info, warn, trace}; use sp_runtime::traits::Header; use sc_service::AbstractService; use std::time::Duration; @@ -36,6 +36,11 @@ pub fn build(service: &impl AbstractService) -> impl futures::Future { + at: std::time::Instant, + value: T, +} + +/// Some value frozen for period of time. +/// +/// If time `duration` not passed since the value was instantiated, +/// current frozen value is returned. Otherwise, you have to provide +/// a new value which will be again frozen for `duration`. +pub(crate) struct FrozenForDuration { + duration: std::time::Duration, + value: RwLock>, +} + +impl FrozenForDuration { + fn new(duration: std::time::Duration, initial: T) -> Self { + Self { + duration, + value: Frozen { at: std::time::Instant::now(), value: initial }.into(), + } + } + + fn take_or_else(&self, f: F) -> T where F: FnOnce() -> T { + if self.value.read().at.elapsed() > self.duration { + let mut write_lock = self.value.write(); + let new_value = f(); + write_lock.at = std::time::Instant::now(); + write_lock.value = new_value.clone(); + new_value + } else { + self.value.read().value.clone() + } + } +} + +/// Disk backend. +/// +/// Disk backend keps data in a key-value store. In archive mode, trie nodes are kept from all blocks. /// Otherwise, trie nodes are kept only from some recent blocks. pub struct Backend { storage: Arc>, @@ -845,6 +886,7 @@ pub struct Backend { shared_cache: SharedCache, import_lock: RwLock<()>, is_archive: bool, + io_stats: FrozenForDuration, } impl> Backend { @@ -906,6 +948,7 @@ impl> Backend { ), import_lock: Default::default(), is_archive: is_archive_pruning, + io_stats: FrozenForDuration::new(std::time::Duration::from_secs(1), kvdb::IoStats::empty()), }) } @@ -1492,9 +1535,31 @@ impl sc_client_api::backend::Backend for Backend Option { + let io_stats = self.io_stats.take_or_else(|| self.storage.db.io_stats(kvdb::IoStatsKind::SincePrevious)); + let database_cache = parity_util_mem::malloc_size(&*self.storage.db); + let state_cache = (*&self.shared_cache).lock().used_storage_cache_size(); + + Some(UsageInfo { + memory: MemoryInfo { + state_cache, + database_cache, + }, + io: IoInfo { + transactions: io_stats.transactions, + bytes_read: io_stats.bytes_read, + bytes_written: io_stats.bytes_written, + writes: io_stats.writes, + reads: io_stats.reads, + average_transaction_size: io_stats.avg_transaction_size() as u64, + }, + }) + } + fn revert(&self, n: NumberFor, revert_finalized: bool) -> ClientResult> { let mut best_number = self.blockchain.info().best_number; let mut best_hash = self.blockchain.info().best_hash; + let finalized = self.blockchain.info().finalized_number; let revertible = best_number - finalized; @@ -1563,11 +1628,6 @@ impl sc_client_api::backend::Backend for Backend Option { - let used = (*&self.shared_cache).lock().used_storage_cache_size(); - Some(used) - } - fn state_at(&self, block: BlockId) -> ClientResult { use sc_client::blockchain::HeaderBackend as BcHeaderBackend; diff --git a/substrate/client/db/src/light.rs b/substrate/client/db/src/light.rs index 7533ec9d7a..2885ee5046 100644 --- a/substrate/client/db/src/light.rs +++ b/substrate/client/db/src/light.rs @@ -22,7 +22,7 @@ use parking_lot::RwLock; use kvdb::{KeyValueDB, DBTransaction}; -use sc_client_api::backend::{AuxStore, NewBlockState}; +use sc_client_api::{backend::{AuxStore, NewBlockState}, UsageInfo}; use sc_client::blockchain::{ BlockStatus, Cache as BlockchainCache,Info as BlockchainInfo, }; @@ -30,7 +30,7 @@ use sc_client::cht; use sp_blockchain::{ CachedHeaderMetadata, HeaderMetadata, HeaderMetadataCache, Error as ClientError, Result as ClientResult, - HeaderBackend as BlockchainHeaderBackend, + HeaderBackend as BlockchainHeaderBackend, well_known_cache_keys, }; use sc_client::light::blockchain::Storage as LightBlockchainStorage; @@ -40,7 +40,7 @@ use sp_runtime::generic::{DigestItem, BlockId}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, One, NumberFor}; use crate::cache::{DbCacheSync, DbCache, ComplexBlockId, EntryType as CacheEntryType}; use crate::utils::{self, meta_keys, Meta, db_err, read_db, block_id_to_lookup_key, read_meta}; -use crate::DatabaseSettings; +use crate::{DatabaseSettings, FrozenForDuration}; use log::{trace, warn, debug}; pub(crate) mod columns { @@ -64,6 +64,7 @@ pub struct LightStorage { meta: RwLock, Block::Hash>>, cache: Arc>, header_metadata_cache: HeaderMetadataCache, + io_stats: FrozenForDuration, } impl LightStorage @@ -102,6 +103,7 @@ impl LightStorage meta: RwLock::new(meta), cache: Arc::new(DbCacheSync(RwLock::new(cache))), header_metadata_cache: HeaderMetadataCache::default(), + io_stats: FrozenForDuration::new(std::time::Duration::from_secs(1), kvdb::IoStats::empty()), }) } @@ -548,6 +550,28 @@ impl LightBlockchainStorage for LightStorage fn cache(&self) -> Option>> { Some(self.cache.clone()) } + + fn usage_info(&self) -> Option { + use sc_client_api::{MemoryInfo, IoInfo}; + + let database_cache = parity_util_mem::malloc_size(&*self.db); + let io_stats = self.io_stats.take_or_else(|| self.db.io_stats(kvdb::IoStatsKind::SincePrevious)); + + Some(UsageInfo { + memory: MemoryInfo { + database_cache, + state_cache: 0, + }, + io: IoInfo { + transactions: io_stats.transactions, + bytes_read: io_stats.bytes_read, + bytes_written: io_stats.bytes_written, + writes: io_stats.writes, + reads: io_stats.reads, + average_transaction_size: io_stats.avg_transaction_size() as u64, + } + }) + } } /// Build the key for inserting header-CHT at given block. diff --git a/substrate/client/service/src/builder.rs b/substrate/client/service/src/builder.rs index e456efe365..8a4d760acc 100644 --- a/substrate/client/service/src/builder.rs +++ b/substrate/client/service/src/builder.rs @@ -932,11 +932,6 @@ ServiceBuilder< let bandwidth_download = net_status.average_download_per_sec; let bandwidth_upload = net_status.average_upload_per_sec; - let used_state_cache_size = match info.used_state_cache_size { - Some(size) => size, - None => 0, - }; - // get cpu usage and memory usage of this process let (cpu_usage, memory) = if let Some(self_pid) = self_pid { if sys.refresh_process(self_pid) { @@ -959,7 +954,10 @@ ServiceBuilder< "finalized_hash" => ?info.chain.finalized_hash, "bandwidth_download" => bandwidth_download, "bandwidth_upload" => bandwidth_upload, - "used_state_cache_size" => used_state_cache_size, + "used_state_cache_size" => info.usage.as_ref().map(|usage| usage.memory.state_cache).unwrap_or(0), + "used_db_cache_size" => info.usage.as_ref().map(|usage| usage.memory.database_cache).unwrap_or(0), + "disk_read_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_read).unwrap_or(0), + "disk_write_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_written).unwrap_or(0), ); let _ = record_metrics!( "peers" => num_peers, @@ -970,7 +968,10 @@ ServiceBuilder< "finalized_height" => finalized_number, "bandwidth_download" => bandwidth_download, "bandwidth_upload" => bandwidth_upload, - "used_state_cache_size" => used_state_cache_size, + "used_state_cache_size" => info.usage.as_ref().map(|usage| usage.memory.state_cache).unwrap_or(0), + "used_db_cache_size" => info.usage.as_ref().map(|usage| usage.memory.database_cache).unwrap_or(0), + "disk_read_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_read).unwrap_or(0), + "disk_write_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_written).unwrap_or(0), ); Ok(()) diff --git a/substrate/client/src/client.rs b/substrate/client/src/client.rs index 8376abd026..f99789de7d 100644 --- a/substrate/client/src/client.rs +++ b/substrate/client/src/client.rs @@ -1176,7 +1176,7 @@ impl Client where pub fn usage_info(&self) -> ClientInfo { ClientInfo { chain: self.chain_info(), - used_state_cache_size: self.backend.used_state_cache_size(), + usage: self.backend.usage_info(), } } diff --git a/substrate/client/src/in_mem.rs b/substrate/client/src/in_mem.rs index 020f2b9e4e..5a54960aa6 100644 --- a/substrate/client/src/in_mem.rs +++ b/substrate/client/src/in_mem.rs @@ -37,6 +37,7 @@ use sc_client_api::{ blockchain::{ self, BlockStatus, HeaderBackend, well_known_cache_keys::Id as CacheKeyId }, + UsageInfo, }; use crate::leaves::LeafSet; @@ -449,6 +450,10 @@ impl sc_client_api::light::Storage for Blockchain fn cache(&self) -> Option>> { None } + + fn usage_info(&self) -> Option { + None + } } /// In-memory operation. @@ -681,7 +686,7 @@ where &self.blockchain } - fn used_state_cache_size(&self) -> Option { + fn usage_info(&self) -> Option { None } diff --git a/substrate/client/src/light/backend.rs b/substrate/client/src/light/backend.rs index 568d290834..3f680e6243 100644 --- a/substrate/client/src/light/backend.rs +++ b/substrate/client/src/light/backend.rs @@ -39,6 +39,7 @@ use sc_client_api::{ HeaderBackend as BlockchainHeaderBackend, well_known_cache_keys, }, light::Storage as BlockchainStorage, + UsageInfo, }; use crate::light::blockchain::Blockchain; use hash_db::Hasher; @@ -186,8 +187,8 @@ impl ClientBackend for Backend where &self.blockchain } - fn used_state_cache_size(&self) -> Option { - None + fn usage_info(&self) -> Option { + self.blockchain.storage().usage_info() } fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage> { diff --git a/substrate/client/src/light/blockchain.rs b/substrate/client/src/light/blockchain.rs index 9e8d14a48b..1ea4987078 100644 --- a/substrate/client/src/light/blockchain.rs +++ b/substrate/client/src/light/blockchain.rs @@ -314,5 +314,9 @@ pub mod tests { fn cache(&self) -> Option>> { None } + + fn usage_info(&self) -> Option { + None + } } } diff --git a/substrate/utils/browser/Cargo.toml b/substrate/utils/browser/Cargo.toml index 861aae4962..c7703474b7 100644 --- a/substrate/utils/browser/Cargo.toml +++ b/substrate/utils/browser/Cargo.toml @@ -15,7 +15,7 @@ console_log = "0.1.2" js-sys = "0.3.34" wasm-bindgen = "0.2.57" wasm-bindgen-futures = "0.4.7" -kvdb-web = "0.2" +kvdb-web = "0.3" service = { version = "2.0.0", package = "sc-service", path = "../../client/service", default-features = false } network = { package = "sc-network", path = "../../client/network" } chain-spec = { package = "sc-chain-spec", path = "../../client/chain-spec" }