pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-05-30 07:01:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Gather memory usage statistics through parity-util-mem (#3893)

Browse Source 
* Gather memory usage statistics through `parity-util-mem`

* Update `Cargo.lock`
This commit is contained in:
Koute
2021-09-20 17:51:34 +09:00
committed by GitHub
parent fa4e5131fb
commit 7b39f096b8
9 changed files with 37 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
polkadot/Cargo.lock
Generated
+4 -3
View File
@@ -5480,14 +5480,15 @@ dependencies = [
[[package]]
name = "parity-util-mem"
version = "0.10.0"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ad6f1acec69b95caf435bbd158d486e5a0a44fcf51531e84922c59ff09e8457"
checksum = "9f7adaf50e545c285006d384d50588e98c405c49b55c0aa05660aca081f6ee5e"
dependencies = [
"cfg-if 1.0.0",
"ethereum-types",
"hashbrown",
"impl-trait-for-tuples",
"jemalloc-ctl",
"jemallocator",
"lru",
"parity-util-mem-derive",
@@ -6395,7 +6396,6 @@ version = "0.9.9"
dependencies = [
"futures 0.3.17",
"futures-timer 3.0.2",
"jemalloc-ctl",
"metered-channel",
"substrate-prometheus-endpoint",
]
@@ -6524,6 +6524,7 @@ dependencies = [
"futures-timer 3.0.2",
"lru",
"metered-channel",
"parity-util-mem",
"parking_lot 0.11.1",
"polkadot-node-metrics",
"polkadot-node-network-protocol",
polkadot/cli/Cargo.toml
-7
View File
@@ -46,13 +46,6 @@ cli = [
"frame-benchmarking-cli",
"try-runtime-cli",
"polkadot-node-core-pvf",
# memory stats require jemalloc, which we know is enabled for Linux
# but not present on wasm or windows
# https://github.com/paritytech/parity-common/blob/master/parity-util-mem/src/allocators.rs#L9-L34
# Once
# https://github.com/rust-lang/cargo/issues/1197
# is resolved.
"service/memory-stats",
]
runtime-benchmarks = [ "service/runtime-benchmarks" ]
trie-memory-tracker = [ "sp-trie/memory-tracker" ]
polkadot/node/metrics/Cargo.toml
-3
View File
@@ -13,8 +13,5 @@ metered-channel = { path = "../metered-channel" }
substrate-prometheus-endpoint = { git = "https://github.com/paritytech/substrate", branch = "master" }
jemalloc-ctl = { version = "0.3.3", optional = true }
[features]
default = []
memory-stats = ["jemalloc-ctl"]
polkadot/node/metrics/src/lib.rs
-7
View File
@@ -26,13 +26,6 @@
pub use metered_channel as metered;
/// Memory allocation stats tracking.
#[cfg(feature = "memory-stats")]
pub mod memory_stats;
#[cfg(feature = "memory-stats")]
pub use self::memory_stats::{MemoryAllocationSnapshot, MemoryAllocationTracker};
/// Cyclic metric collection support.
pub mod metronome;
pub use self::metronome::Metronome;
polkadot/node/metrics/src/memory_stats.rs
-66
View File
@@ -1,66 +0,0 @@
// Copyright 2021 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Memory tracking statistics.
//!
//! Many subsystems have common interests such as canceling a bunch of spawned jobs,
//! or determining what their validator ID is. These common interests are factored into
//! this module.
//!
//! This crate also reexports Prometheus metric types which are expected to be implemented by subsystems.
// #[cfg(not(feature = "memory-stats"))]
// use std::convert::Infallible;
use jemalloc_ctl::{epoch, stats, Result};
/// Accessor to the allocator internals.
#[derive(Clone)]
pub struct MemoryAllocationTracker {
epoch: jemalloc_ctl::epoch_mib,
allocated: stats::allocated_mib,
resident: stats::resident_mib,
}
impl MemoryAllocationTracker {
/// Create an instance of an allocation tracker.
pub fn new() -> Result<Self> {
Ok(Self {
epoch: epoch::mib()?,
allocated: stats::allocated::mib()?,
resident: stats::resident::mib()?,
})
}
/// Create an allocation snapshot.
pub fn snapshot(&self) -> Result<MemoryAllocationSnapshot> {
// update stats by advancing the allocation epoch
self.epoch.advance()?;
let allocated: u64 = self.allocated.read()? as _;
let resident: u64 = self.resident.read()? as _;
Ok(MemoryAllocationSnapshot { allocated, resident })
}
}
/// Snapshot of collected memory metrics.
#[derive(Debug, Clone)]
pub struct MemoryAllocationSnapshot {
/// Total resident memory, in bytes.
pub resident: u64,
/// Total allocated memory, in bytes.
pub allocated: u64,
}
polkadot/node/overseer/Cargo.toml
+1 -1
View File
@@ -19,6 +19,7 @@ polkadot-overseer-gen = { path = "./overseer-gen" }
polkadot-overseer-all-subsystems-gen = { path = "./all-subsystems-gen" }
tracing = "0.1.27"
lru = "0.6"
parity-util-mem = { version = ">= 0.10.1", default-features = false }
[dev-dependencies]
metered-channel = { path = "../metered-channel" }
@@ -29,4 +30,3 @@ assert_matches = "1.4.0"
[features]
default = []
memory-stats = ["polkadot-node-metrics/memory-stats"]
polkadot/node/overseer/src/lib.rs
+15 -5
View File
@@ -108,8 +108,7 @@ use polkadot_node_metrics::{
Metronome,
};
#[cfg(feature = "memory-stats")]
use polkadot_node_metrics::memory_stats::MemoryAllocationTracker;
use parity_util_mem::MemoryAllocationTracker;
pub use polkadot_overseer_gen as gen;
pub use polkadot_overseer_gen::{
@@ -649,13 +648,23 @@ where
}
let subsystem_meters = overseer.map_subsystems(ExtractNameAndMeters);
#[cfg(feature = "memory-stats")]
let memory_stats = MemoryAllocationTracker::new().expect("Jemalloc is the default allocator. qed");
let memory_stats = match MemoryAllocationTracker::new() {
Ok(memory_stats) => Some(memory_stats),
Err(error) => {
tracing::debug!(
target: LOG_TARGET,
"Failed to initialize memory allocation tracker: {:?}",
error
);
None
},
};
let metronome_metrics = metrics.clone();
let metronome =
Metronome::new(std::time::Duration::from_millis(950)).for_each(move |_| {
#[cfg(feature = "memory-stats")]
if let Some(ref memory_stats) = memory_stats {
match memory_stats.snapshot() {
Ok(memory_stats_snapshot) => {
tracing::trace!(
@@ -672,6 +681,7 @@ where
e
),
}
}
// We combine the amount of messages from subsystems to the overseer
// as well as the amount of messages from external sources to the overseer
polkadot/node/overseer/src/metrics.rs
+3 -13
View File
@@ -19,8 +19,7 @@
use super::*;
pub use polkadot_node_metrics::metrics::{self, prometheus, Metrics as MetricsTrait};
#[cfg(feature = "memory-stats")]
use polkadot_node_metrics::MemoryAllocationSnapshot;
use parity_util_mem::MemoryAllocationSnapshot;
/// Overseer Prometheus metrics.
#[derive(Clone)]
@@ -35,10 +34,7 @@ struct MetricsInner {
signals_sent: prometheus::GaugeVec<prometheus::U64>,
signals_received: prometheus::GaugeVec<prometheus::U64>,
#[cfg(feature = "memory-stats")]
memory_stats_resident: prometheus::Gauge<prometheus::U64>,
#[cfg(feature = "memory-stats")]
memory_stats_allocated: prometheus::Gauge<prometheus::U64>,
}
@@ -65,13 +61,10 @@ impl Metrics {
}
}
#[cfg(feature = "memory-stats")]
pub(crate) fn memory_stats_snapshot(&self, memory_stats: MemoryAllocationSnapshot) {
if let Some(metrics) = &self.0 {
let MemoryAllocationSnapshot { resident, allocated } = memory_stats;
metrics.memory_stats_allocated.set(allocated);
metrics.memory_stats_resident.set(resident);
metrics.memory_stats_allocated.set(memory_stats.allocated);
metrics.memory_stats_resident.set(memory_stats.resident);
}
}
@@ -202,7 +195,6 @@ impl MetricsTrait for Metrics {
registry,
)?,
#[cfg(feature = "memory-stats")]
memory_stats_allocated: prometheus::register(
prometheus::Gauge::<prometheus::U64>::new(
"memory_allocated",
@@ -210,8 +202,6 @@ impl MetricsTrait for Metrics {
)?,
registry,
)?,
#[cfg(feature = "memory-stats")]
memory_stats_resident: prometheus::register(
prometheus::Gauge::<prometheus::U64>::new(
"memory_resident",
polkadot/node/service/Cargo.toml
-1
View File
@@ -175,5 +175,4 @@ try-runtime = [
"rococo-runtime/try-runtime",
]
malus = ["full-node"]
memory-stats = ["polkadot-overseer/memory-stats"]
disputes = ["polkadot-node-core-dispute-coordinator/disputes"]