diff --git a/substrate/Cargo.lock b/substrate/Cargo.lock index 440b3b218e..ca7a5d1844 100644 --- a/substrate/Cargo.lock +++ b/substrate/Cargo.lock @@ -2527,31 +2527,31 @@ dependencies = [ [[package]] name = "kvdb" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8396be0e5561ccd1bf7ff0b2007487cdd7a87a056873fe6ea906b35d4dbf7ed8" +checksum = "03080afe6f42cd996da9f568d6add5d7fb5ee2ea7fb7802d2d2cbd836958fd87" dependencies = [ "parity-bytes", - "parity-util-mem 0.4.1", + "parity-util-mem 0.5.1", "smallvec 1.2.0", ] [[package]] name = "kvdb-memorydb" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d25ef14155e418515c4839e9144c839de3506e68946f255a32b7f166095493d" +checksum = "b9355274e5a9e0a7e8ef43916950eae3949024de2a8dffe4d5a6c13974a37c8e" dependencies = [ "kvdb", - "parity-util-mem 0.4.1", - "parking_lot 0.9.0", + "parity-util-mem 0.5.1", + "parking_lot 0.10.0", ] [[package]] name = "kvdb-rocksdb" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af488cc16c3801705c8d681c3a32c8faa8fafc7fb5309dee0f573f3c6a19d395" +checksum = "af36fd66ccd99f3f771ae39b75aaba28b952372b6debfb971134bf1f03466ab2" dependencies = [ "fs-swap", "interleaved-ordered", @@ -2559,8 +2559,8 @@ dependencies = [ "log 0.4.8", "num_cpus", "owning_ref", - "parity-util-mem 0.4.1", - "parking_lot 0.9.0", + "parity-util-mem 0.5.1", + "parking_lot 0.10.0", "regex", "rocksdb", "smallvec 1.2.0", @@ -2568,16 +2568,16 @@ dependencies = [ [[package]] name = "kvdb-web" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a0e36637fb86454de401e7cb88f40eb0ad1b9bcee837d46785e7c451f1ebf4" +checksum = "7a985c47b4c46429e96033ebf6eaed784a81ceccb4e5df13d63f3b9078a4df81" dependencies = [ "futures 0.3.1", "js-sys", "kvdb", "kvdb-memorydb", "log 0.4.8", - "parity-util-mem 0.4.1", + "parity-util-mem 0.5.1", "send_wrapper 0.3.0", "wasm-bindgen", "web-sys", @@ -4620,14 +4620,15 @@ dependencies = [ [[package]] name = "parity-util-mem" -version = "0.4.1" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900dd84654b048e5bad420bb341658fc2c4d7fea628c22bcf4621733e54859b4" +checksum = "ef1476e40bf8f5c6776e9600983435821ca86eb9819d74a6207cca69d091406a" dependencies = [ "cfg-if", "impl-trait-for-tuples", "parity-util-mem-derive", - "parking_lot 0.9.0", + "parking_lot 0.10.0", + "primitive-types", "smallvec 1.2.0", "winapi 0.3.8", ] @@ -5629,6 +5630,7 @@ dependencies = [ "lazy_static", "log 0.4.8", "names", + "parity-util-mem 0.5.1", "regex", "rpassword", "sc-client-api", @@ -5729,7 +5731,7 @@ dependencies = [ "linked-hash-map", "log 0.4.8", "parity-scale-codec", - "parity-util-mem 0.4.1", + "parity-util-mem 0.5.1", "parking_lot 0.10.0", "quickcheck", "sc-client", @@ -6308,6 +6310,7 @@ dependencies = [ "log 0.4.8", "parity-multiaddr 0.5.0", "parity-scale-codec", + "parity-util-mem 0.5.1", "parking_lot 0.10.0", "sc-chain-spec", "sc-client", @@ -6422,6 +6425,7 @@ dependencies = [ "futures 0.3.1", "log 0.4.8", "parity-scale-codec", + "parity-util-mem 0.5.1", "parking_lot 0.10.0", "serde", "sp-core", @@ -6439,6 +6443,7 @@ dependencies = [ "futures-diagnose", "log 0.4.8", "parity-scale-codec", + "parity-util-mem 0.5.1", "parking_lot 0.10.0", "sc-client-api", "sc-transaction-graph", @@ -7001,6 +7006,7 @@ dependencies = [ "log 0.4.8", "num-traits", "parity-scale-codec", + "parity-util-mem 0.5.1", "parking_lot 0.10.0", "pretty_assertions", "primitive-types", @@ -7146,6 +7152,7 @@ dependencies = [ "impl-trait-for-tuples", "log 0.4.8", "parity-scale-codec", + "parity-util-mem 0.5.1", "paste", "rand 0.7.3", "serde", @@ -7290,6 +7297,7 @@ name = "sp-test-primitives" version = "2.0.0" dependencies = [ "parity-scale-codec", + "parity-util-mem 0.5.1", "serde", "sp-application-crypto", "sp-core", @@ -7571,6 +7579,7 @@ dependencies = [ "pallet-babe", "pallet-timestamp", "parity-scale-codec", + "parity-util-mem 0.5.1", "sc-client", "sc-executor", "serde", diff --git a/substrate/client/Cargo.toml b/substrate/client/Cargo.toml index 8560408569..c89fe88145 100644 --- a/substrate/client/Cargo.toml +++ b/substrate/client/Cargo.toml @@ -19,7 +19,7 @@ hash-db = { version = "0.15.2" } hex-literal = { version = "0.2.1" } sp-inherents = { version = "2.0.0", path = "../primitives/inherents" } sp-keyring = { version = "2.0.0", path = "../primitives/keyring" } -kvdb = "0.3.0" +kvdb = "0.4.0" log = { version = "0.4.8" } parking_lot = "0.10.0" sp-core = { version = "2.0.0", path = "../primitives/core" } @@ -37,5 +37,5 @@ tracing = "0.1.10" env_logger = "0.7.0" tempfile = "3.1.0" substrate-test-runtime-client = { version = "2.0.0", path = "../test-utils/runtime/client" } -kvdb-memorydb = "0.3.0" +kvdb-memorydb = "0.4.0" sp-panic-handler = { version = "2.0.0", path = "../primitives/panic-handler" } diff --git a/substrate/client/api/Cargo.toml b/substrate/client/api/Cargo.toml index 4c2867deb3..27a40c4d94 100644 --- a/substrate/client/api/Cargo.toml +++ b/substrate/client/api/Cargo.toml @@ -18,7 +18,7 @@ sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } hex-literal = { version = "0.2.1" } sp-inherents = { version = "2.0.0", default-features = false, path = "../../primitives/inherents" } sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } -kvdb = "0.3.0" +kvdb = "0.4.0" log = { version = "0.4.8" } parking_lot = "0.10.0" sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } diff --git a/substrate/client/cli/Cargo.toml b/substrate/client/cli/Cargo.toml index abfb53ccb1..e302d53d55 100644 --- a/substrate/client/cli/Cargo.toml +++ b/substrate/client/cli/Cargo.toml @@ -35,6 +35,7 @@ names = "0.11.0" structopt = "0.3.8" sc-tracing = { version = "2.0.0", path = "../tracing" } chrono = "0.4.10" +parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } [target.'cfg(not(target_os = "unknown"))'.dependencies] rpassword = "4.0.1" diff --git a/substrate/client/cli/src/informant.rs b/substrate/client/cli/src/informant.rs index 312e4017d5..9e7c5044e0 100644 --- a/substrate/client/cli/src/informant.rs +++ b/substrate/client/cli/src/informant.rs @@ -28,6 +28,7 @@ mod display; /// Creates an informant in the form of a `Future` that must be polled regularly. pub fn build(service: &impl AbstractService) -> impl futures::Future { let client = service.client(); + let pool = service.transaction_pool(); let mut display = display::InformantDisplay::new(); @@ -40,6 +41,11 @@ pub fn build(service: &impl AbstractService) -> impl futures::Future info.usage.as_ref().map(|usage| usage.memory.database_cache).unwrap_or(0), "disk_read_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_read).unwrap_or(0), "disk_write_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_written).unwrap_or(0), + "memory_transaction_pool" => parity_util_mem::malloc_size(&*transaction_pool_), ); ready(()) }); + let _ = to_spawn_tx.unbounded_send(( Box::pin(select(tel_task, exit.clone()).map(drop)), From::from("telemetry-periodic-send") diff --git a/substrate/client/transaction-pool/Cargo.toml b/substrate/client/transaction-pool/Cargo.toml index 1387d1df27..524e9a98a0 100644 --- a/substrate/client/transaction-pool/Cargo.toml +++ b/substrate/client/transaction-pool/Cargo.toml @@ -19,6 +19,7 @@ sc-transaction-graph = { version = "2.0.0", path = "./graph" } sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" } sc-client-api = { version = "2.0.0", path = "../api" } sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } +parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } [dev-dependencies] sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } diff --git a/substrate/client/transaction-pool/graph/Cargo.toml b/substrate/client/transaction-pool/graph/Cargo.toml index 4f12ab7fcc..2d3172fc91 100644 --- a/substrate/client/transaction-pool/graph/Cargo.toml +++ b/substrate/client/transaction-pool/graph/Cargo.toml @@ -14,6 +14,7 @@ serde = { version = "1.0.101", features = ["derive"] } sp-core = { version = "2.0.0", path = "../../../primitives/core" } sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } sp-transaction-pool = { version = "2.0.0", path = "../../../primitives/transaction-pool" } +parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } [dev-dependencies] assert_matches = "1.3.0" diff --git a/substrate/client/transaction-pool/graph/src/base_pool.rs b/substrate/client/transaction-pool/graph/src/base_pool.rs index 7b7900c3e9..52e00df363 100644 --- a/substrate/client/transaction-pool/graph/src/base_pool.rs +++ b/substrate/client/transaction-pool/graph/src/base_pool.rs @@ -84,7 +84,7 @@ pub struct PruneStatus { /// Immutable transaction #[cfg_attr(test, derive(Clone))] -#[derive(PartialEq, Eq)] +#[derive(PartialEq, Eq, parity_util_mem::MallocSizeOf)] pub struct Transaction { /// Raw extrinsic representing that transaction. pub data: Extrinsic, @@ -209,7 +209,7 @@ const RECENTLY_PRUNED_TAGS: usize = 2; /// as-is for the second time will fail or produce unwanted results. /// Most likely it is required to revalidate them and recompute set of /// required tags. -#[derive(Debug)] +#[derive(Debug, parity_util_mem::MallocSizeOf)] pub struct BasePool { reject_future_transactions: bool, future: FutureTransactions, @@ -846,6 +846,33 @@ mod tests { } } + #[test] + fn can_track_heap_size() { + let mut pool = pool(); + pool.import(Transaction { + data: vec![5u8; 1024], + bytes: 1, + hash: 5, + priority: 5u64, + valid_till: 64u64, + requires: vec![], + provides: vec![vec![0], vec![4]], + propagate: true, + }).expect("import 1 should be ok"); + pool.import(Transaction { + data: vec![3u8; 1024], + bytes: 1, + hash: 7, + priority: 5u64, + valid_till: 64u64, + requires: vec![], + provides: vec![vec![2], vec![7]], + propagate: true, + }).expect("import 2 should be ok"); + + assert!(parity_util_mem::malloc_size(&pool) > 5000); + } + #[test] fn should_remove_invalid_transactions() { // given diff --git a/substrate/client/transaction-pool/graph/src/future.rs b/substrate/client/transaction-pool/graph/src/future.rs index 0de50c1a65..bda26fe34f 100644 --- a/substrate/client/transaction-pool/graph/src/future.rs +++ b/substrate/client/transaction-pool/graph/src/future.rs @@ -29,6 +29,7 @@ use sp_runtime::transaction_validity::{ use crate::base_pool::Transaction; +#[derive(parity_util_mem::MallocSizeOf)] /// Transaction with partially satisfied dependencies. pub struct WaitingTransaction { /// Transaction details. @@ -109,7 +110,7 @@ impl WaitingTransaction { /// /// Contains transactions that are still awaiting for some other transactions that /// could provide a tag that they require. -#[derive(Debug)] +#[derive(Debug, parity_util_mem::MallocSizeOf)] pub struct FutureTransactions { /// tags that are not yet provided by any transaction and we await for them wanted_tags: HashMap>, @@ -243,3 +244,30 @@ impl FutureTransactions { self.waiting.values().fold(0, |acc, tx| acc + tx.transaction.bytes) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn can_track_heap_size() { + let mut future = FutureTransactions::default(); + future.import(WaitingTransaction { + transaction: Transaction { + data: vec![0u8; 1024], + bytes: 1, + hash: 1, + priority: 1, + valid_till: 2, + requires: vec![vec![1], vec![2]], + provides: vec![vec![3], vec![4]], + propagate: true, + }.into(), + missing_tags: vec![vec![1u8], vec![2u8]].into_iter().collect(), + imported_at: std::time::Instant::now(), + }); + + // data is at least 1024! + assert!(parity_util_mem::malloc_size(&future) > 1024); + } +} diff --git a/substrate/client/transaction-pool/graph/src/pool.rs b/substrate/client/transaction-pool/graph/src/pool.rs index 91ce58518a..ab4e3a5a79 100644 --- a/substrate/client/transaction-pool/graph/src/pool.rs +++ b/substrate/client/transaction-pool/graph/src/pool.rs @@ -122,6 +122,16 @@ pub struct Pool { validated_pool: Arc>, } +impl parity_util_mem::MallocSizeOf for Pool +where + B::Hash: parity_util_mem::MallocSizeOf, + ExtrinsicFor: parity_util_mem::MallocSizeOf, +{ + fn size_of(&self, ops: &mut parity_util_mem::MallocSizeOfOps) -> usize { + self.validated_pool.size_of(ops) + } +} + impl Pool { /// Create a new transaction pool. pub fn new(options: Options, api: Arc) -> Self { diff --git a/substrate/client/transaction-pool/graph/src/ready.rs b/substrate/client/transaction-pool/graph/src/ready.rs index ec8d66e6b9..23f0d49a93 100644 --- a/substrate/client/transaction-pool/graph/src/ready.rs +++ b/substrate/client/transaction-pool/graph/src/ready.rs @@ -36,7 +36,7 @@ use crate::base_pool::Transaction; /// An in-pool transaction reference. /// /// Should be cheap to clone. -#[derive(Debug)] +#[derive(Debug, parity_util_mem::MallocSizeOf)] pub struct TransactionRef { /// The actual transaction data. pub transaction: Arc>, @@ -74,7 +74,7 @@ impl PartialEq for TransactionRef { } impl Eq for TransactionRef {} -#[derive(Debug)] +#[derive(Debug, parity_util_mem::MallocSizeOf)] pub struct ReadyTx { /// A reference to a transaction pub transaction: TransactionRef, @@ -104,7 +104,7 @@ Hence every hash retrieved from `provided_tags` is always present in `ready`; qed "#; -#[derive(Debug)] +#[derive(Debug, parity_util_mem::MallocSizeOf)] pub struct ReadyTransactions { /// Insertion id insertion_id: u64, @@ -676,6 +676,24 @@ mod tests { assert_eq!(it.next(), None); } + #[test] + fn can_report_heap_size() { + let mut ready = ReadyTransactions::default(); + let tx = Transaction { + data: vec![5], + bytes: 1, + hash: 5, + priority: 1, + valid_till: u64::max_value(), // use the max_value() here for testing. + requires: vec![], + provides: vec![], + propagate: true, + }; + import(&mut ready, tx).unwrap(); + + assert!(parity_util_mem::malloc_size(&ready) > 200); + } + #[test] fn should_order_refs() { let mut id = 1; diff --git a/substrate/client/transaction-pool/graph/src/validated_pool.rs b/substrate/client/transaction-pool/graph/src/validated_pool.rs index c7e60deb4a..9524284064 100644 --- a/substrate/client/transaction-pool/graph/src/validated_pool.rs +++ b/substrate/client/transaction-pool/graph/src/validated_pool.rs @@ -74,6 +74,17 @@ pub(crate) struct ValidatedPool { rotator: PoolRotator>, } +impl parity_util_mem::MallocSizeOf for ValidatedPool +where + B::Hash: parity_util_mem::MallocSizeOf, + ExtrinsicFor: parity_util_mem::MallocSizeOf, +{ + fn size_of(&self, ops: &mut parity_util_mem::MallocSizeOfOps) -> usize { + // other entries insignificant or non-primary references + self.pool.size_of(ops) + } +} + impl ValidatedPool { /// Create a new transaction pool. pub fn new(options: Options, api: Arc) -> Self { diff --git a/substrate/client/transaction-pool/src/lib.rs b/substrate/client/transaction-pool/src/lib.rs index 85bf2fd327..7084e1c4a0 100644 --- a/substrate/client/transaction-pool/src/lib.rs +++ b/substrate/client/transaction-pool/src/lib.rs @@ -53,6 +53,18 @@ pub struct BasicPool revalidation_strategy: Arc>>>, } +impl parity_util_mem::MallocSizeOf for BasicPool +where + PoolApi: sc_transaction_graph::ChainApi, + PoolApi::Hash: parity_util_mem::MallocSizeOf, + Block: BlockT, +{ + fn size_of(&self, ops: &mut parity_util_mem::MallocSizeOfOps) -> usize { + // other entries insignificant or non-primary references + self.pool.size_of(ops) + } +} + /// Type of revalidation. pub enum RevalidationType { /// Light revalidation type. diff --git a/substrate/client/transaction-pool/src/testing/pool.rs b/substrate/client/transaction-pool/src/testing/pool.rs index d4f3d0ccb4..fed02067b1 100644 --- a/substrate/client/transaction-pool/src/testing/pool.rs +++ b/substrate/client/transaction-pool/src/testing/pool.rs @@ -215,3 +215,14 @@ fn should_not_retain_invalid_hashes_from_retracted() { block_on(pool.maintain(&BlockId::number(1), &[retracted_hash])); assert_eq!(pool.status().ready, 0); } + +#[test] +fn can_track_heap_size() { + let pool = maintained_pool(); + block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).expect("1. Imported"); + block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 210))).expect("1. Imported"); + block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 211))).expect("1. Imported"); + block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 212))).expect("1. Imported"); + + assert!(parity_util_mem::malloc_size(&pool) > 3000); +} \ No newline at end of file diff --git a/substrate/frame/system/src/lib.rs b/substrate/frame/system/src/lib.rs index 4a06dec6ee..a0f3700c9d 100644 --- a/substrate/frame/system/src/lib.rs +++ b/substrate/frame/system/src/lib.rs @@ -106,7 +106,7 @@ use sp_runtime::{ traits::{ self, CheckEqual, SimpleArithmetic, Zero, SignedExtension, Lookup, LookupError, SimpleBitOps, Hash, Member, MaybeDisplay, EnsureOrigin, BadOrigin, SaturatedConversion, - MaybeSerialize, MaybeSerializeDeserialize, StaticLookup, One, Bounded, + MaybeSerialize, MaybeSerializeDeserialize, MaybeMallocSizeOf, StaticLookup, One, Bounded, }, }; @@ -171,12 +171,12 @@ pub trait Trait: 'static + Eq + Clone { /// The block number type used by the runtime. type BlockNumber: Parameter + Member + MaybeSerializeDeserialize + Debug + MaybeDisplay + SimpleArithmetic - + Default + Bounded + Copy + sp_std::hash::Hash + sp_std::str::FromStr; + + Default + Bounded + Copy + sp_std::hash::Hash + sp_std::str::FromStr + MaybeMallocSizeOf; /// The output of the `Hashing` function. type Hash: Parameter + Member + MaybeSerializeDeserialize + Debug + MaybeDisplay + SimpleBitOps + Ord - + Default + Copy + CheckEqual + sp_std::hash::Hash + AsRef<[u8]> + AsMut<[u8]>; + + Default + Copy + CheckEqual + sp_std::hash::Hash + AsRef<[u8]> + AsMut<[u8]> + MaybeMallocSizeOf; /// The hashing system (algorithm) being used in the runtime (e.g. Blake2). type Hashing: Hash; diff --git a/substrate/primitives/core/Cargo.toml b/substrate/primitives/core/Cargo.toml index 873d607ecd..9bd30d1823 100644 --- a/substrate/primitives/core/Cargo.toml +++ b/substrate/primitives/core/Cargo.toml @@ -30,6 +30,7 @@ sp-debug-derive = { version = "2.0.0", path = "../debug-derive" } sp-externalities = { version = "0.8.0", optional = true, path = "../externalities" } sp-storage = { version = "2.0.0", default-features = false, path = "../storage" } libsecp256k1 = { version = "0.3.2", default-features = false } +parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } # full crypto ed25519-dalek = { version = "1.0.0-pre.3", default-features = false, features = ["u64_backend", "alloc"], optional = true } diff --git a/substrate/primitives/core/src/changes_trie.rs b/substrate/primitives/core/src/changes_trie.rs index d38761ccf0..cb21ffe13d 100644 --- a/substrate/primitives/core/src/changes_trie.rs +++ b/substrate/primitives/core/src/changes_trie.rs @@ -22,7 +22,7 @@ use codec::{Encode, Decode}; use num_traits::Zero; /// Substrate changes trie configuration. -#[cfg_attr(any(feature = "std", test), derive(Serialize, Deserialize))] +#[cfg_attr(any(feature = "std", test), derive(Serialize, Deserialize, parity_util_mem::MallocSizeOf))] #[derive(Debug, Clone, PartialEq, Eq, Default, Encode, Decode)] pub struct ChangesTrieConfiguration { /// Interval (in blocks) at which level1-digests are created. Digests are not diff --git a/substrate/primitives/runtime/Cargo.toml b/substrate/primitives/runtime/Cargo.toml index 6f50a1a4d9..bf7b2b80a3 100644 --- a/substrate/primitives/runtime/Cargo.toml +++ b/substrate/primitives/runtime/Cargo.toml @@ -18,6 +18,7 @@ paste = "0.1.6" rand = { version = "0.7.2", optional = true } impl-trait-for-tuples = "0.1.3" sp-inherents = { version = "2.0.0", default-features = false, path = "../inherents" } +parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } [dev-dependencies] serde_json = "1.0.41" @@ -37,4 +38,5 @@ std = [ "sp-io/std", "serde", "sp-inherents/std", + "parity-util-mem/std", ] diff --git a/substrate/primitives/runtime/src/generic/block.rs b/substrate/primitives/runtime/src/generic/block.rs index 21e65d1fb5..a46396dce0 100644 --- a/substrate/primitives/runtime/src/generic/block.rs +++ b/substrate/primitives/runtime/src/generic/block.rs @@ -25,7 +25,7 @@ use serde::{Deserialize, Serialize}; use sp_std::prelude::*; use sp_core::RuntimeDebug; use crate::codec::{Codec, Encode, Decode}; -use crate::traits::{self, Member, Block as BlockT, Header as HeaderT, MaybeSerialize}; +use crate::traits::{self, Member, Block as BlockT, Header as HeaderT, MaybeSerialize, MaybeMallocSizeOf}; use crate::Justification; /// Something to identify a block. @@ -63,7 +63,7 @@ impl fmt::Display for BlockId { /// Abstraction over a substrate block. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, parity_util_mem::MallocSizeOf))] #[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] #[cfg_attr(feature = "std", serde(deny_unknown_fields))] pub struct Block { @@ -76,7 +76,7 @@ pub struct Block { impl traits::Block for Block where Header: HeaderT, - Extrinsic: Member + Codec + traits::Extrinsic, + Extrinsic: Member + Codec + traits::Extrinsic + MaybeMallocSizeOf, { type Extrinsic = Extrinsic; type Header = Header; diff --git a/substrate/primitives/runtime/src/generic/digest.rs b/substrate/primitives/runtime/src/generic/digest.rs index fef02d4f00..4d09b58793 100644 --- a/substrate/primitives/runtime/src/generic/digest.rs +++ b/substrate/primitives/runtime/src/generic/digest.rs @@ -27,7 +27,7 @@ use sp_core::{ChangesTrieConfiguration, RuntimeDebug}; /// Generic header digest. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, parity_util_mem::MallocSizeOf))] pub struct Digest { /// A list of logs in the digest. pub logs: Vec>, @@ -74,6 +74,7 @@ impl Digest { /// Digest item that is able to encode/decode 'system' digest items and /// provide opaque access to other items. #[derive(PartialEq, Eq, Clone, RuntimeDebug)] +#[cfg_attr(feature = "std", derive(parity_util_mem::MallocSizeOf))] pub enum DigestItem { /// System digest item that contains the root of changes trie at given /// block. It is created for every block iff runtime supports changes @@ -107,7 +108,7 @@ pub enum DigestItem { /// Available changes trie signals. #[derive(PartialEq, Eq, Clone, Encode, Decode)] -#[cfg_attr(feature = "std", derive(Debug))] +#[cfg_attr(feature = "std", derive(Debug, parity_util_mem::MallocSizeOf))] pub enum ChangesTrieSignal { /// New changes trie configuration is enacted, starting from **next block**. /// diff --git a/substrate/primitives/runtime/src/generic/header.rs b/substrate/primitives/runtime/src/generic/header.rs index 51f31af078..5bc7932feb 100644 --- a/substrate/primitives/runtime/src/generic/header.rs +++ b/substrate/primitives/runtime/src/generic/header.rs @@ -22,6 +22,7 @@ use crate::codec::{Decode, Encode, Codec, Input, Output, HasCompact, EncodeAsRef use crate::traits::{ self, Member, SimpleArithmetic, SimpleBitOps, Hash as HashT, MaybeSerializeDeserialize, MaybeSerialize, MaybeDisplay, + MaybeMallocSizeOf, }; use crate::generic::Digest; use sp_core::U256; @@ -51,6 +52,22 @@ pub struct Header + TryFrom, Hash: HashT> { pub digest: Digest, } +#[cfg(feature = "std")] +impl parity_util_mem::MallocSizeOf for Header +where + Number: Copy + Into + TryFrom + parity_util_mem::MallocSizeOf, + Hash: HashT, + Hash::Output: parity_util_mem::MallocSizeOf, +{ + fn size_of(&self, ops: &mut parity_util_mem::MallocSizeOfOps) -> usize { + self.parent_hash.size_of(ops) + + self.number.size_of(ops) + + self.state_root.size_of(ops) + + self.extrinsics_root.size_of(ops) + + self.digest.size_of(ops) + } +} + #[cfg(feature = "std")] pub fn serialize_number + TryFrom>( val: &T, s: S, @@ -105,10 +122,11 @@ impl codec::EncodeLike for Header where impl traits::Header for Header where Number: Member + MaybeSerializeDeserialize + Debug + sp_std::hash::Hash + MaybeDisplay + - SimpleArithmetic + Codec + Copy + Into + TryFrom + sp_std::str::FromStr, + SimpleArithmetic + Codec + Copy + Into + TryFrom + sp_std::str::FromStr + + MaybeMallocSizeOf, Hash: HashT, Hash::Output: Default + sp_std::hash::Hash + Copy + Member + Ord + - MaybeSerialize + Debug + MaybeDisplay + SimpleBitOps + Codec, + MaybeSerialize + Debug + MaybeDisplay + SimpleBitOps + Codec + MaybeMallocSizeOf, { type Number = Number; type Hash = ::Output; diff --git a/substrate/primitives/runtime/src/generic/unchecked_extrinsic.rs b/substrate/primitives/runtime/src/generic/unchecked_extrinsic.rs index 1625d42e93..a516bc1f7f 100644 --- a/substrate/primitives/runtime/src/generic/unchecked_extrinsic.rs +++ b/substrate/primitives/runtime/src/generic/unchecked_extrinsic.rs @@ -44,6 +44,18 @@ where pub function: Call, } +#[cfg(feature = "std")] +impl parity_util_mem::MallocSizeOf + for UncheckedExtrinsic +where + Extra: SignedExtension +{ + fn size_of(&self, _ops: &mut parity_util_mem::MallocSizeOfOps) -> usize { + // Instantiated only in runtime. + 0 + } +} + impl UncheckedExtrinsic { diff --git a/substrate/primitives/runtime/src/lib.rs b/substrate/primitives/runtime/src/lib.rs index 46930c35e8..8d8effcc66 100644 --- a/substrate/primitives/runtime/src/lib.rs +++ b/substrate/primitives/runtime/src/lib.rs @@ -639,6 +639,13 @@ macro_rules! assert_eq_error_rate { #[derive(PartialEq, Eq, Clone, Default, Encode, Decode)] pub struct OpaqueExtrinsic(pub Vec); +#[cfg(feature = "std")] +impl parity_util_mem::MallocSizeOf for OpaqueExtrinsic { + fn size_of(&self, ops: &mut parity_util_mem::MallocSizeOfOps) -> usize { + self.0.size_of(ops) + } +} + impl sp_std::fmt::Debug for OpaqueExtrinsic { #[cfg(feature = "std")] fn fmt(&self, fmt: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { diff --git a/substrate/primitives/runtime/src/testing.rs b/substrate/primitives/runtime/src/testing.rs index c86638b57b..e3e94c3c9f 100644 --- a/substrate/primitives/runtime/src/testing.rs +++ b/substrate/primitives/runtime/src/testing.rs @@ -148,7 +148,7 @@ pub type DigestItem = generic::DigestItem; pub type Digest = generic::Digest; /// Block Header -#[derive(PartialEq, Eq, Clone, Serialize, Debug, Encode, Decode, Default)] +#[derive(PartialEq, Eq, Clone, Serialize, Debug, Encode, Decode, Default, parity_util_mem::MallocSizeOf)] #[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct Header { @@ -220,10 +220,12 @@ impl<'a> Deserialize<'a> for Header { } /// An opaque extrinsic wrapper type. -#[derive(PartialEq, Eq, Clone, Debug, Encode, Decode)] +#[derive(PartialEq, Eq, Clone, Debug, Encode, Decode, parity_util_mem::MallocSizeOf)] pub struct ExtrinsicWrapper(Xt); -impl traits::Extrinsic for ExtrinsicWrapper { +impl traits::Extrinsic for ExtrinsicWrapper +where Xt: parity_util_mem::MallocSizeOf +{ type Call = (); type SignaturePayload = (); @@ -253,7 +255,7 @@ impl Deref for ExtrinsicWrapper { } /// Testing block -#[derive(PartialEq, Eq, Clone, Serialize, Debug, Encode, Decode)] +#[derive(PartialEq, Eq, Clone, Serialize, Debug, Encode, Decode, parity_util_mem::MallocSizeOf)] pub struct Block { /// Block header pub header: Header, @@ -300,6 +302,9 @@ impl<'a, Xt> Deserialize<'a> for Block where Block: Decode { #[derive(PartialEq, Eq, Clone, Encode, Decode)] pub struct TestXt(pub Option<(u64, Extra)>, pub Call); +// Non-opaque extrinsics always 0. +parity_util_mem::malloc_size_of_is_0!(any: TestXt); + impl Serialize for TestXt where TestXt: Encode { fn serialize(&self, seq: S) -> Result where S: Serializer { self.using_encoded(|bytes| seq.serialize_bytes(bytes)) diff --git a/substrate/primitives/runtime/src/traits.rs b/substrate/primitives/runtime/src/traits.rs index f8c2ed11a4..f07f4dd7ac 100644 --- a/substrate/primitives/runtime/src/traits.rs +++ b/substrate/primitives/runtime/src/traits.rs @@ -470,6 +470,9 @@ sp_core::impl_maybe_marker!( /// A type that implements Serialize, DeserializeOwned and Debug when in std environment. trait MaybeSerializeDeserialize: DeserializeOwned, Serialize; + + /// A type that implements MallocSizeOf. + trait MaybeMallocSizeOf: parity_util_mem::MallocSizeOf; ); /// A type that provides a randomness beacon. @@ -503,13 +506,18 @@ pub trait IsMember { /// `parent_hash`, as well as a `digest` and a block `number`. /// /// You can also create a `new` one from those fields. -pub trait Header: Clone + Send + Sync + Codec + Eq + MaybeSerialize + Debug + 'static { +pub trait Header: + Clone + Send + Sync + Codec + Eq + MaybeSerialize + Debug + + MaybeMallocSizeOf + 'static +{ /// Header number. type Number: Member + MaybeSerializeDeserialize + Debug + sp_std::hash::Hash - + Copy + MaybeDisplay + SimpleArithmetic + Codec + sp_std::str::FromStr; + + Copy + MaybeDisplay + SimpleArithmetic + Codec + sp_std::str::FromStr + + MaybeMallocSizeOf; /// Header hash type type Hash: Member + MaybeSerializeDeserialize + Debug + sp_std::hash::Hash + Ord - + Copy + MaybeDisplay + Default + SimpleBitOps + Codec + AsRef<[u8]> + AsMut<[u8]>; + + Copy + MaybeDisplay + Default + SimpleBitOps + Codec + AsRef<[u8]> + + AsMut<[u8]> + MaybeMallocSizeOf; /// Hashing algorithm type Hashing: Hash; @@ -557,14 +565,15 @@ pub trait Header: Clone + Send + Sync + Codec + Eq + MaybeSerialize + Debug + 's /// `Extrinsic` pieces of information as well as a `Header`. /// /// You can get an iterator over each of the `extrinsics` and retrieve the `header`. -pub trait Block: Clone + Send + Sync + Codec + Eq + MaybeSerialize + Debug + 'static { +pub trait Block: Clone + Send + Sync + Codec + Eq + MaybeSerialize + Debug + MaybeMallocSizeOf + 'static { /// Type for extrinsics. - type Extrinsic: Member + Codec + Extrinsic + MaybeSerialize; + type Extrinsic: Member + Codec + Extrinsic + MaybeSerialize + MaybeMallocSizeOf; /// Header type. - type Header: Header; + type Header: Header + MaybeMallocSizeOf; /// Block hash type. type Hash: Member + MaybeSerializeDeserialize + Debug + sp_std::hash::Hash + Ord - + Copy + MaybeDisplay + Default + SimpleBitOps + Codec + AsRef<[u8]> + AsMut<[u8]>; + + Copy + MaybeDisplay + Default + SimpleBitOps + Codec + AsRef<[u8]> + AsMut<[u8]> + + MaybeMallocSizeOf; /// Returns a reference to the header. fn header(&self) -> &Self::Header; @@ -583,8 +592,9 @@ pub trait Block: Clone + Send + Sync + Codec + Eq + MaybeSerialize + Debug + 'st fn encode_from(header: &Self::Header, extrinsics: &[Self::Extrinsic]) -> Vec; } + /// Something that acts like an `Extrinsic`. -pub trait Extrinsic: Sized { +pub trait Extrinsic: Sized + MaybeMallocSizeOf { /// The function call. type Call; diff --git a/substrate/primitives/test-primitives/Cargo.toml b/substrate/primitives/test-primitives/Cargo.toml index ba19eb00f3..2edd5f0575 100644 --- a/substrate/primitives/test-primitives/Cargo.toml +++ b/substrate/primitives/test-primitives/Cargo.toml @@ -11,6 +11,7 @@ codec = { package = "parity-scale-codec", version = "1.0.0", default-features = sp-core = { version = "2.0.0", default-features = false, path = "../core" } serde = { version = "1.0.101", optional = true, features = ["derive"] } sp-runtime = { version = "2.0.0", default-features = false, path = "../runtime" } +parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } [features] default = [ diff --git a/substrate/primitives/test-primitives/src/lib.rs b/substrate/primitives/test-primitives/src/lib.rs index 74115bdb8f..302b24fcc1 100644 --- a/substrate/primitives/test-primitives/src/lib.rs +++ b/substrate/primitives/test-primitives/src/lib.rs @@ -28,6 +28,7 @@ use sp_runtime::traits::{BlakeTwo256, Verify, Extrinsic as ExtrinsicT,}; /// Extrinsic for test-runtime. #[derive(Clone, PartialEq, Eq, Encode, Decode, RuntimeDebug)] +#[cfg_attr(feature = "std", derive(parity_util_mem::MallocSizeOf))] pub enum Extrinsic { IncludeData(Vec), StorageChange(Vec, Option>), diff --git a/substrate/primitives/transaction-pool/src/pool.rs b/substrate/primitives/transaction-pool/src/pool.rs index 2a71773996..0b23c27f82 100644 --- a/substrate/primitives/transaction-pool/src/pool.rs +++ b/substrate/primitives/transaction-pool/src/pool.rs @@ -29,7 +29,7 @@ use futures::{ use serde::{Deserialize, Serialize}; use sp_runtime::{ generic::BlockId, - traits::{Block as BlockT, Member}, + traits::{Block as BlockT, Member, MaybeMallocSizeOf}, transaction_validity::{ TransactionLongevity, TransactionPriority, TransactionTag, }, @@ -154,7 +154,7 @@ pub trait InPoolTransaction { } /// Transaction pool interface. -pub trait TransactionPool: Send + Sync { +pub trait TransactionPool: Send + Sync + MaybeMallocSizeOf { /// Block type. type Block: BlockT; /// Transaction hash type. diff --git a/substrate/test-utils/runtime/Cargo.toml b/substrate/test-utils/runtime/Cargo.toml index e0d8aa77b7..684a681554 100644 --- a/substrate/test-utils/runtime/Cargo.toml +++ b/substrate/test-utils/runtime/Cargo.toml @@ -37,6 +37,7 @@ sc-client = { version = "0.8", optional = true, path = "../../client" } sp-trie = { version = "2.0.0", default-features = false, path = "../../primitives/trie" } sp-transaction-pool = { version = "2.0.0", default-features = false, path = "../../primitives/transaction-pool" } trie-db = { version = "0.19.2", default-features = false } +parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } [dev-dependencies] sc-executor = { version = "0.8", path = "../../client/executor" } diff --git a/substrate/test-utils/runtime/src/lib.rs b/substrate/test-utils/runtime/src/lib.rs index 96387b1efc..dac232b25c 100644 --- a/substrate/test-utils/runtime/src/lib.rs +++ b/substrate/test-utils/runtime/src/lib.rs @@ -114,6 +114,8 @@ pub enum Extrinsic { ChangesTrieConfigUpdate(Option), } +parity_util_mem::malloc_size_of_is_0!(Extrinsic); // non-opaque extrinisic does not need this + #[cfg(feature = "std")] impl serde::Serialize for Extrinsic { fn serialize(&self, seq: S) -> Result where S: ::serde::Serializer { diff --git a/substrate/utils/browser/Cargo.toml b/substrate/utils/browser/Cargo.toml index 36f21f192b..3afa49fc25 100644 --- a/substrate/utils/browser/Cargo.toml +++ b/substrate/utils/browser/Cargo.toml @@ -16,7 +16,7 @@ console_log = "0.1.2" js-sys = "0.3.34" wasm-bindgen = "0.2.57" wasm-bindgen-futures = "0.4.7" -kvdb-web = "0.3" +kvdb-web = "0.4" service = { version = "0.8", package = "sc-service", path = "../../client/service", default-features = false } network = { package = "sc-network", path = "../../client/network" } chain-spec = { package = "sc-chain-spec", path = "../../client/chain-spec" }