Companion for paritytech/substrate#12795 (#6374)

* Begin removing `parity-util-mem`; remove `collect_memory_stats`

* Update some dependencies that were using `parity-util-mem`

* Remove `trie-memory-tracker` feature

* Update Cargo.lock

* Update `kvdb-shared-tests`

* Add back jemalloc

* Add missing license header

* update lockfile for {"substrate"}

Co-authored-by: parity-processbot <>
Co-authored-by: Andronik <write@reusable.software>

polkadot/node/core/approval-voting/Cargo.toml

@@ -13,7 +13,7 @@ bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] }
 lru = "0.8"
 merlin = "2.0"
 schnorrkel = "0.9.1"
-kvdb = "0.12.0"
+kvdb = "0.13.0"
 derive_more = "0.99.17"
 thiserror = "1.0.31"
 
@@ -40,5 +40,5 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" }
 polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
 assert_matches = "1.4.0"
-kvdb-memorydb = "0.12.0"
+kvdb-memorydb = "0.13.0"
 test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" }

polkadot/node/core/av-store/Cargo.toml

@@ -7,7 +7,7 @@ edition.workspace = true
 [dependencies]
 futures = "0.3.21"
 futures-timer = "3.0.2"
-kvdb = "0.12.0"
+kvdb = "0.13.0"
 thiserror = "1.0.31"
 gum = { package = "tracing-gum", path = "../../gum" }
 bitvec = "1.0.0"
@@ -24,7 +24,7 @@ polkadot-node-primitives = { path = "../../primitives" }
 log = "0.4.17"
 env_logger = "0.9.0"
 assert_matches = "1.4.0"
-kvdb-memorydb = "0.12.0"
+kvdb-memorydb = "0.13.0"
 
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 polkadot-node-subsystem-util = { path = "../../subsystem-util" }

polkadot/node/core/chain-selection/Cargo.toml

@@ -13,7 +13,7 @@ polkadot-primitives = { path = "../../../primitives" }
 polkadot-node-primitives = { path = "../../primitives" }
 polkadot-node-subsystem = { path = "../../subsystem" }
 polkadot-node-subsystem-util = { path = "../../subsystem-util" }
-kvdb = "0.12.0"
+kvdb = "0.13.0"
 thiserror = "1.0.31"
 parity-scale-codec = "3.1.5"
 
@@ -22,4 +22,4 @@ polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 parking_lot = "0.12.0"
 assert_matches = "1"
-kvdb-memorydb = "0.12.0"
+kvdb-memorydb = "0.13.0"

polkadot/node/core/dispute-coordinator/Cargo.toml

@@ -8,7 +8,7 @@ edition.workspace = true
 futures = "0.3.21"
 gum = { package = "tracing-gum", path = "../../gum" }
 parity-scale-codec = "3.1.5"
-kvdb = "0.12.0"
+kvdb = "0.13.0"
 thiserror = "1.0.31"
 lru = "0.8.0"
 fatality = "0.0.6"
@@ -22,7 +22,7 @@ sc-keystore = { git = "https://github.com/paritytech/substrate", branch = "maste
 
 
 [dev-dependencies]
-kvdb-memorydb = "0.12.0"
+kvdb-memorydb = "0.13.0"
 polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
 sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }

polkadot/node/malus/Cargo.toml

@@ -23,7 +23,6 @@ polkadot-node-core-backing = { path = "../core/backing" }
 polkadot-node-primitives = { path = "../primitives" }
 polkadot-primitives = { path = "../../primitives" }
 polkadot-node-core-pvf = { path = "../core/pvf" }
-parity-util-mem = { version = "0.12.0", default-features = false, features = ["jemalloc-global"] }
 color-eyre = { version = "0.6.1", default-features = false }
 assert_matches = "1.5"
 async-trait = "0.1.57"

polkadot/node/overseer/Cargo.toml

@@ -18,9 +18,9 @@ polkadot-primitives = { path = "../../primitives" }
 orchestra = "0.0.2"
 gum = { package = "tracing-gum", path = "../gum" }
 lru = "0.8"
-parity-util-mem = { version = "0.12.0", default-features = false }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 async-trait = "0.1.57"
+tikv-jemalloc-ctl = "0.5.0"
 
 [dev-dependencies]
 metered = { package = "prioritized-metered-channel", version = "0.2.0" }

polkadot/node/overseer/src/lib.rs

@@ -101,8 +101,6 @@ pub use polkadot_node_metrics::{
 	Metronome,
 };
 
-use parity_util_mem::MemoryAllocationTracker;
-
 pub use orchestra as gen;
 pub use orchestra::{
 	contextbounds, orchestra, subsystem, FromOrchestra, MapSubsystem, MessagePacket,
@@ -118,11 +116,14 @@ pub const KNOWN_LEAVES_CACHE_SIZE: NonZeroUsize = match NonZeroUsize::new(2 * 24
 	None => panic!("Known leaves cache size must be non-zero"),
 };
 
+mod memory_stats;
 #[cfg(test)]
 mod tests;
 
 use sp_core::traits::SpawnNamed;
 
+use memory_stats::MemoryAllocationTracker;
+
 /// Glue to connect `trait orchestra::Spawner` and `SpawnNamed` from `substrate`.
 pub struct SpawnGlue<S>(pub S);
 

polkadot/node/overseer/src/memory_stats.rs

@@ -0,0 +1,53 @@
+// Copyright 2017-2022 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use tikv_jemalloc_ctl::{epoch, stats, Error};
+
+#[derive(Clone)]
+pub struct MemoryAllocationTracker {
+	epoch: tikv_jemalloc_ctl::epoch_mib,
+	allocated: stats::allocated_mib,
+	resident: stats::resident_mib,
+}
+
+impl MemoryAllocationTracker {
+	pub fn new() -> Result<Self, Error> {
+		Ok(Self {
+			epoch: epoch::mib()?,
+			allocated: stats::allocated::mib()?,
+			resident: stats::resident::mib()?,
+		})
+	}
+
+	pub fn snapshot(&self) -> Result<MemoryAllocationSnapshot, Error> {
+		// update stats by advancing the allocation epoch
+		self.epoch.advance()?;
+
+		let allocated: u64 = self.allocated.read()? as _;
+		let resident: u64 = self.resident.read()? as _;
+		Ok(MemoryAllocationSnapshot { allocated, resident })
+	}
+}
+
+/// Snapshot of collected memory metrics.
+#[non_exhaustive]
+#[derive(Debug, Clone)]
+pub struct MemoryAllocationSnapshot {
+	/// Total resident memory, in bytes.
+	pub resident: u64,
+	/// Total allocated memory, in bytes.
+	pub allocated: u64,
+}

polkadot/node/overseer/src/metrics.rs

@@ -19,7 +19,7 @@
 use super::*;
 pub use polkadot_node_metrics::metrics::{self, prometheus, Metrics as MetricsTrait};
 
-use parity_util_mem::MemoryAllocationSnapshot;
+use memory_stats::MemoryAllocationSnapshot;
 
 /// Overseer Prometheus metrics.
 #[derive(Clone)]

polkadot/node/service/Cargo.toml

@@ -71,8 +71,8 @@ gum = { package = "tracing-gum", path = "../gum/" }
 serde = { version = "1.0.137", features = ["derive"] }
 serde_json = "1.0.81"
 thiserror = "1.0.31"
-kvdb = "0.12.0"
-kvdb-rocksdb = { version = "0.16.0", optional = true }
+kvdb = "0.13.0"
+kvdb-rocksdb = { version = "0.17.0", optional = true }
 parity-db = { version = "0.4.2", optional = true }
 
 async-trait = "0.1.57"

polkadot/node/subsystem-util/Cargo.toml

@@ -32,8 +32,7 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-application-crypto = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
-kvdb = "0.12.0"
-parity-util-mem = { version = "0.12.0", default-features = false }
+kvdb = "0.13.0"
 parity-db = { version = "0.4.2"}
 
 [dev-dependencies]
@@ -44,6 +43,6 @@ log = "0.4.17"
 polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" }
 lazy_static = "1.4.0"
 polkadot-primitives-test-helpers = { path = "../../primitives/test-helpers" }
-kvdb-shared-tests = "0.10.0"
+kvdb-shared-tests = "0.11.0"
 tempfile = "3.1.0"
-kvdb-memorydb = "0.12.0"
+kvdb-memorydb = "0.13.0"

polkadot/node/subsystem-util/src/database.rs

@@ -29,7 +29,6 @@ pub trait Database: KeyValueDB {
 pub mod kvdb_impl {
 	use super::{DBKeyValue, DBTransaction, DBValue, Database, KeyValueDB};
 	use kvdb::{DBOp, IoStats, IoStatsKind};
-	use parity_util_mem::{MallocSizeOf, MallocSizeOfOps};
 	use std::{collections::BTreeSet, io::Result};
 
 	/// Adapter implementing subsystem database
@@ -123,13 +122,6 @@ pub mod kvdb_impl {
 			self.db.has_prefix(col, prefix)
 		}
 	}
-
-	impl<D: KeyValueDB> MallocSizeOf for DbAdapter<D> {
-		fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize {
-			// ignore filter set
-			self.db.size_of(ops)
-		}
-	}
 }
 
 /// Utilities for using parity-db database.
@@ -160,12 +152,6 @@ pub mod paritydb_impl {
 		write_lock: Arc<Mutex<()>>,
 	}
 
-	impl parity_util_mem::MallocSizeOf for DbAdapter {
-		fn size_of(&self, _ops: &mut parity_util_mem::MallocSizeOfOps) -> usize {
-			unimplemented!("size_of is not supported for parity_db")
-		}
-	}
-
 	impl KeyValueDB for DbAdapter {
 		fn transaction(&self) -> DBTransaction {
 			DBTransaction::new()