Additional Metrics collected and exposed via prometheus (#5414)

This PR refactors the metrics measuring and Prometheus exposing entity in sc-service into its own submodule and extends the parameters it exposes by:

- system load average (over one, five and 15min)
- the TCP connection state of the process (lsof), refs #5304
- number of tokio threads
- number of known forks
- counter for items in each unbounded queue (with internal unbounded channels)
- number of file descriptors opened by this process (*nix only at this point)
- number of system threads (*nix only at this point)

refs #4679

Co-authored-by: Max Inden <mail@max-inden.de>
Co-authored-by: Ashley <ashley.ruglys@gmail.com>

substrate/client/finality-grandpa/src/communication/gossip.rs

@@ -90,7 +90,7 @@ use sp_finality_grandpa::AuthorityId;
 
 use sc_telemetry::{telemetry, CONSENSUS_DEBUG};
 use log::{trace, debug};
-use futures::channel::mpsc;
+use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender};
 use prometheus_endpoint::{CounterVec, Opts, PrometheusError, register, Registry, U64};
 use rand::seq::SliceRandom;
 
@@ -1254,7 +1254,7 @@ impl Metrics {
 pub(super) struct GossipValidator<Block: BlockT> {
 	inner: parking_lot::RwLock<Inner<Block>>,
 	set_state: environment::SharedVoterSetState<Block>,
-	report_sender: mpsc::UnboundedSender<PeerReport>,
+	report_sender: TracingUnboundedSender<PeerReport>,
 	metrics: Option<Metrics>,
 }
 
@@ -1266,7 +1266,7 @@ impl<Block: BlockT> GossipValidator<Block> {
 		config: crate::Config,
 		set_state: environment::SharedVoterSetState<Block>,
 		prometheus_registry: Option<&Registry>,
-	) -> (GossipValidator<Block>, mpsc::UnboundedReceiver<PeerReport>)	{
+	) -> (GossipValidator<Block>, TracingUnboundedReceiver<PeerReport>)	{
 		let metrics = match prometheus_registry.map(Metrics::register) {
 			Some(Ok(metrics)) => Some(metrics),
 			Some(Err(e)) => {
@@ -1276,7 +1276,7 @@ impl<Block: BlockT> GossipValidator<Block> {
 			None => None,
 		};
 
-		let (tx, rx) = mpsc::unbounded();
+		let (tx, rx) = tracing_unbounded("mpsc_grandpa_gossip_validator");
 		let val = GossipValidator {
 			inner: parking_lot::RwLock::new(Inner::new(config)),
 			set_state,

substrate/client/finality-grandpa/src/communication/mod.rs

@@ -58,6 +58,7 @@ use gossip::{
 use sp_finality_grandpa::{
 	AuthorityPair, AuthorityId, AuthoritySignature, SetId as SetIdNumber, RoundNumber,
 };
+use sp_utils::mpsc::TracingUnboundedReceiver;
 
 pub mod gossip;
 mod periodic;
@@ -165,7 +166,7 @@ pub(crate) struct NetworkBridge<B: BlockT, N: Network<B>> {
 	// thus one has to wrap gossip_validator_report_stream with an `Arc` `Mutex`. Given that it is
 	// just an `UnboundedReceiver`, one could also switch to a multi-producer-*multi*-consumer
 	// channel implementation.
-	gossip_validator_report_stream: Arc<Mutex<mpsc::UnboundedReceiver<PeerReport>>>,
+	gossip_validator_report_stream: Arc<Mutex<TracingUnboundedReceiver<PeerReport>>>,
 }
 
 impl<B: BlockT, N: Network<B>> Unpin for NetworkBridge<B, N> {}

substrate/client/finality-grandpa/src/communication/periodic.rs

@@ -17,9 +17,10 @@
 //! Periodic rebroadcast of neighbor packets.
 
 use futures_timer::Delay;
-use futures::{channel::mpsc, future::{FutureExt as _}, prelude::*, ready, stream::Stream};
+use futures::{future::{FutureExt as _}, prelude::*, ready, stream::Stream};
 use log::debug;
 use std::{pin::Pin, task::{Context, Poll}, time::Duration};
+use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender};
 
 use sc_network::PeerId;
 use sp_runtime::traits::{NumberFor, Block as BlockT};
@@ -31,7 +32,7 @@ const REBROADCAST_AFTER: Duration = Duration::from_secs(2 * 60);
 /// A sender used to send neighbor packets to a background job.
 #[derive(Clone)]
 pub(super) struct NeighborPacketSender<B: BlockT>(
-	mpsc::UnboundedSender<(Vec<PeerId>, NeighborPacket<NumberFor<B>>)>
+	TracingUnboundedSender<(Vec<PeerId>, NeighborPacket<NumberFor<B>>)>
 );
 
 impl<B: BlockT> NeighborPacketSender<B> {
@@ -54,14 +55,15 @@ impl<B: BlockT> NeighborPacketSender<B> {
 pub(super) struct NeighborPacketWorker<B: BlockT> {
 	last: Option<(Vec<PeerId>, NeighborPacket<NumberFor<B>>)>,
 	delay: Delay,
-	rx: mpsc::UnboundedReceiver<(Vec<PeerId>, NeighborPacket<NumberFor<B>>)>,
+	rx: TracingUnboundedReceiver<(Vec<PeerId>, NeighborPacket<NumberFor<B>>)>,
 }
 
 impl<B: BlockT> Unpin for NeighborPacketWorker<B> {}
 
 impl<B: BlockT> NeighborPacketWorker<B> {
 	pub(super) fn new() -> (Self, NeighborPacketSender<B>){
-		let (tx, rx) = mpsc::unbounded::<(Vec<PeerId>, NeighborPacket<NumberFor<B>>)>();
+		let (tx, rx) = tracing_unbounded::<(Vec<PeerId>, NeighborPacket<NumberFor<B>>)>
+			("mpsc_grandpa_neighbor_packet_worker");
 		let delay = Delay::new(REBROADCAST_AFTER);
 
 		(NeighborPacketWorker {

substrate/client/finality-grandpa/src/communication/tests.rs

@@ -16,7 +16,7 @@
 
 //! Tests for the communication portion of the GRANDPA crate.
 
-use futures::channel::mpsc;
+use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender};
 use futures::prelude::*;
 use sc_network::{Event as NetworkEvent, ObservedRole, PeerId};
 use sc_network_test::{Block, Hash};
@@ -33,7 +33,7 @@ use super::{AuthorityId, VoterSet, Round, SetId};
 
 #[derive(Debug)]
 pub(crate) enum Event {
-	EventStream(mpsc::UnboundedSender<NetworkEvent>),
+	EventStream(TracingUnboundedSender<NetworkEvent>),
 	WriteNotification(sc_network::PeerId, Vec<u8>),
 	Report(sc_network::PeerId, sc_network::ReputationChange),
 	Announce(Hash),
@@ -41,12 +41,12 @@ pub(crate) enum Event {
 
 #[derive(Clone)]
 pub(crate) struct TestNetwork {
-	sender: mpsc::UnboundedSender<Event>,
+	sender: TracingUnboundedSender<Event>,
 }
 
 impl sc_network_gossip::Network<Block> for TestNetwork {
 	fn event_stream(&self) -> Pin<Box<dyn Stream<Item = NetworkEvent> + Send>> {
-		let (tx, rx) = mpsc::unbounded();
+		let (tx, rx) = tracing_unbounded("test");
 		let _ = self.sender.unbounded_send(Event::EventStream(tx));
 		Box::pin(rx)
 	}
@@ -97,7 +97,7 @@ impl sc_network_gossip::ValidatorContext<Block> for TestNetwork {
 pub(crate) struct Tester {
 	pub(crate) net_handle: super::NetworkBridge<Block, TestNetwork>,
 	gossip_validator: Arc<GossipValidator<Block>>,
-	pub(crate) events: mpsc::UnboundedReceiver<Event>,
+	pub(crate) events: TracingUnboundedReceiver<Event>,
 }
 
 impl Tester {
@@ -161,7 +161,7 @@ pub(crate) fn make_test_network() -> (
 	impl Future<Output = Tester>,
 	TestNetwork,
 ) {
-	let (tx, rx) = mpsc::unbounded();
+	let (tx, rx) = tracing_unbounded("test");
 	let net = TestNetwork { sender: tx };
 
 	#[derive(Clone)]

substrate/client/finality-grandpa/src/import.rs

@@ -18,11 +18,11 @@ use std::{sync::Arc, collections::HashMap};
 
 use log::{debug, trace, info};
 use parity_scale_codec::Encode;
-use futures::channel::mpsc;
 use parking_lot::RwLockWriteGuard;
 
 use sp_blockchain::{BlockStatus, well_known_cache_keys};
 use sc_client_api::{backend::Backend, utils::is_descendent_of};
+use sp_utils::mpsc::TracingUnboundedSender;
 use sp_api::{TransactionFor};
 
 use sp_consensus::{
@@ -57,7 +57,7 @@ pub struct GrandpaBlockImport<Backend, Block: BlockT, Client, SC> {
 	inner: Arc<Client>,
 	select_chain: SC,
 	authority_set: SharedAuthoritySet<Block::Hash, NumberFor<Block>>,
-	send_voter_commands: mpsc::UnboundedSender<VoterCommand<Block::Hash, NumberFor<Block>>>,
+	send_voter_commands: TracingUnboundedSender<VoterCommand<Block::Hash, NumberFor<Block>>>,
 	consensus_changes: SharedConsensusChanges<Block::Hash, NumberFor<Block>>,
 	authority_set_hard_forks: HashMap<Block::Hash, PendingChange<Block::Hash, NumberFor<Block>>>,
 	_phantom: PhantomData<Backend>,
@@ -536,7 +536,7 @@ impl<Backend, Block: BlockT, Client, SC> GrandpaBlockImport<Backend, Block, Clie
 		inner: Arc<Client>,
 		select_chain: SC,
 		authority_set: SharedAuthoritySet<Block::Hash, NumberFor<Block>>,
-		send_voter_commands: mpsc::UnboundedSender<VoterCommand<Block::Hash, NumberFor<Block>>>,
+		send_voter_commands: TracingUnboundedSender<VoterCommand<Block::Hash, NumberFor<Block>>>,
 		consensus_changes: SharedConsensusChanges<Block::Hash, NumberFor<Block>>,
 		authority_set_hard_forks: Vec<(SetId, PendingChange<Block::Hash, NumberFor<Block>>)>,
 	) -> GrandpaBlockImport<Backend, Block, Client, SC> {

substrate/client/finality-grandpa/src/lib.rs

@@ -55,7 +55,6 @@
 use futures::prelude::*;
 use futures::StreamExt;
 use log::{debug, info};
-use futures::channel::mpsc;
 use sc_client_api::{
 	backend::{AuxStore, Backend},
 	LockImportRun, BlockchainEvents, CallExecutor,
@@ -70,6 +69,7 @@ use sc_keystore::KeyStorePtr;
 use sp_inherents::InherentDataProviders;
 use sp_consensus::{SelectChain, BlockImport};
 use sp_core::Pair;
+use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver};
 use sc_telemetry::{telemetry, CONSENSUS_INFO, CONSENSUS_DEBUG};
 use serde_json;
 
@@ -379,7 +379,7 @@ pub struct LinkHalf<Block: BlockT, C, SC> {
 	client: Arc<C>,
 	select_chain: SC,
 	persistent_data: PersistentData<Block>,
-	voter_commands_rx: mpsc::UnboundedReceiver<VoterCommand<Block::Hash, NumberFor<Block>>>,
+	voter_commands_rx: TracingUnboundedReceiver<VoterCommand<Block::Hash, NumberFor<Block>>>,
 }
 
 /// Provider for the Grandpa authority set configured on the genesis block.
@@ -476,7 +476,7 @@ where
 		}
 	)?;
 
-	let (voter_commands_tx, voter_commands_rx) = mpsc::unbounded();
+	let (voter_commands_tx, voter_commands_rx) = tracing_unbounded("mpsc_grandpa_voter_command");
 
 	// create pending change objects with 0 delay and enacted on finality
 	// (i.e. standard changes) for each authority set hard fork.
@@ -598,7 +598,7 @@ pub struct GrandpaParams<Block: BlockT, C, N, SC, VR> {
 	/// The inherent data providers.
 	pub inherent_data_providers: InherentDataProviders,
 	/// If supplied, can be used to hook on telemetry connection established events.
-	pub telemetry_on_connect: Option<futures::channel::mpsc::UnboundedReceiver<()>>,
+	pub telemetry_on_connect: Option<TracingUnboundedReceiver<()>>,
 	/// A voting rule used to potentially restrict target votes.
 	pub voting_rule: VR,
 	/// The prometheus metrics registry.
@@ -718,7 +718,7 @@ impl Metrics {
 struct VoterWork<B, Block: BlockT, C, N: NetworkT<Block>, SC, VR> {
 	voter: Pin<Box<dyn Future<Output = Result<(), CommandOrError<Block::Hash, NumberFor<Block>>>> + Send>>,
 	env: Arc<Environment<B, Block, C, N, SC, VR>>,
-	voter_commands_rx: mpsc::UnboundedReceiver<VoterCommand<Block::Hash, NumberFor<Block>>>,
+	voter_commands_rx: TracingUnboundedReceiver<VoterCommand<Block::Hash, NumberFor<Block>>>,
 	network: NetworkBridge<Block, N>,
 
 	/// Prometheus metrics.
@@ -742,7 +742,7 @@ where
 		select_chain: SC,
 		voting_rule: VR,
 		persistent_data: PersistentData<Block>,
-		voter_commands_rx: mpsc::UnboundedReceiver<VoterCommand<Block::Hash, NumberFor<Block>>>,
+		voter_commands_rx: TracingUnboundedReceiver<VoterCommand<Block::Hash, NumberFor<Block>>>,
 		prometheus_registry: Option<prometheus_endpoint::Registry>,
 	) -> Self {
 		let metrics = match prometheus_registry.as_ref().map(Metrics::register) {

substrate/client/finality-grandpa/src/observer.rs

@@ -18,7 +18,7 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::task::{Context, Poll};
 
-use futures::{prelude::*, channel::mpsc};
+use futures::prelude::*;
 
 use finality_grandpa::{
 	BlockNumberOps, Error as GrandpaError, voter, voter_set::VoterSet
@@ -27,8 +27,10 @@ use log::{debug, info, warn};
 
 use sp_consensus::SelectChain;
 use sc_client_api::backend::Backend;
+use sp_utils::mpsc::TracingUnboundedReceiver;
 use sp_runtime::traits::{NumberFor, Block as BlockT};
 use sp_blockchain::HeaderMetadata;
+
 use crate::{
 	global_communication, CommandOrError, CommunicationIn, Config, environment,
 	LinkHalf, Error, aux_schema::PersistentData, VoterCommand, VoterSetState,
@@ -206,7 +208,7 @@ struct ObserverWork<B: BlockT, BE, Client, N: NetworkT<B>> {
 	network: NetworkBridge<B, N>,
 	persistent_data: PersistentData<B>,
 	keystore: Option<sc_keystore::KeyStorePtr>,
-	voter_commands_rx: mpsc::UnboundedReceiver<VoterCommand<B::Hash, NumberFor<B>>>,
+	voter_commands_rx: TracingUnboundedReceiver<VoterCommand<B::Hash, NumberFor<B>>>,
 	_phantom: PhantomData<BE>,
 }
 
@@ -223,7 +225,7 @@ where
 		network: NetworkBridge<B, Network>,
 		persistent_data: PersistentData<B>,
 		keystore: Option<sc_keystore::KeyStorePtr>,
-		voter_commands_rx: mpsc::UnboundedReceiver<VoterCommand<B::Hash, NumberFor<B>>>,
+		voter_commands_rx: TracingUnboundedReceiver<VoterCommand<B::Hash, NumberFor<B>>>,
 	) -> Self {
 
 		let mut work = ObserverWork {
@@ -376,6 +378,7 @@ mod tests {
 	use super::*;
 
 	use assert_matches::assert_matches;
+	use sp_utils::mpsc::tracing_unbounded;
 	use crate::{aux_schema,	communication::tests::{Event, make_test_network}};
 	use substrate_test_runtime_client::{TestClientBuilder, TestClientBuilderExt};
 	use sc_network::PeerId;
@@ -412,7 +415,7 @@ mod tests {
 			|| Ok(vec![]),
 		).unwrap();
 
-		let (_tx, voter_command_rx) = mpsc::unbounded();
+		let (_tx, voter_command_rx) = tracing_unbounded("");
 		let observer = ObserverWork::new(
 			client,
 			tester.net_handle.clone(),

substrate/client/finality-grandpa/src/tests.rs

@@ -993,7 +993,7 @@ fn voter_persists_its_votes() {
 	use std::iter::FromIterator;
 	use std::sync::atomic::{AtomicUsize, Ordering};
 	use futures::future;
-	use futures::channel::mpsc;
+	use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver};
 
 	let _ = env_logger::try_init();
 	let mut runtime = Runtime::new().unwrap();
@@ -1018,7 +1018,7 @@ fn voter_persists_its_votes() {
 
 	// channel between the voter and the main controller.
 	// sending a message on the `voter_tx` restarts the voter.
-	let (voter_tx, voter_rx) = mpsc::unbounded::<()>();
+	let (voter_tx, voter_rx) = tracing_unbounded::<()>("");
 
 	let mut keystore_paths = Vec::new();
 
@@ -1031,7 +1031,7 @@ fn voter_persists_its_votes() {
 
 		struct ResettableVoter {
 			voter: Pin<Box<dyn Future<Output = ()> + Send + Unpin>>,
-			voter_rx: mpsc::UnboundedReceiver<()>,
+			voter_rx: TracingUnboundedReceiver<()>,
 			net: Arc<Mutex<GrandpaTestNet>>,
 			client: PeersClient,
 			keystore: KeyStorePtr,

substrate/client/finality-grandpa/src/until_imported.rs

@@ -29,10 +29,10 @@ use super::{
 };
 
 use log::{debug, warn};
+use sp_utils::mpsc::TracingUnboundedReceiver;
 use futures::prelude::*;
 use futures::stream::Fuse;
 use futures_timer::Delay;
-use futures::channel::mpsc::UnboundedReceiver;
 use finality_grandpa::voter;
 use parking_lot::Mutex;
 use prometheus_endpoint::{
@@ -140,7 +140,7 @@ impl Drop for Metrics {
 /// Buffering imported messages until blocks with given hashes are imported.
 #[pin_project::pin_project]
 pub(crate) struct UntilImported<Block: BlockT, BlockStatus, BlockSyncRequester, I, M: BlockUntilImported<Block>> {
-	import_notifications: Fuse<UnboundedReceiver<BlockImportNotification<Block>>>,
+	import_notifications: Fuse<TracingUnboundedReceiver<BlockImportNotification<Block>>>,
 	block_sync_requester: BlockSyncRequester,
 	status_check: BlockStatus,
 	#[pin]
@@ -541,18 +541,18 @@ mod tests {
 	use sc_client_api::BlockImportNotification;
 	use futures::future::Either;
 	use futures_timer::Delay;
-	use futures::channel::mpsc;
+	use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender};
 	use finality_grandpa::Precommit;
 
 	#[derive(Clone)]
 	struct TestChainState {
-		sender: mpsc::UnboundedSender<BlockImportNotification<Block>>,
+		sender: TracingUnboundedSender<BlockImportNotification<Block>>,
 		known_blocks: Arc<Mutex<HashMap<Hash, u64>>>,
 	}
 
 	impl TestChainState {
 		fn new() -> (Self, ImportNotifications<Block>) {
-			let (tx, rx) = mpsc::unbounded();
+			let (tx, rx) = tracing_unbounded("test");
 			let state = TestChainState {
 				sender: tx,
 				known_blocks: Arc::new(Mutex::new(HashMap::new())),
@@ -649,7 +649,7 @@ mod tests {
 		// enact all dependencies before importing the message
 		enact_dependencies(&chain_state);
 
-		let (global_tx, global_rx) = futures::channel::mpsc::unbounded();
+		let (global_tx, global_rx) = tracing_unbounded("test");
 
 		let until_imported = UntilGlobalMessageBlocksImported::new(
 			import_notifications,
@@ -676,7 +676,7 @@ mod tests {
 		let (chain_state, import_notifications) = TestChainState::new();
 		let block_status = chain_state.block_status();
 
-		let (global_tx, global_rx) = futures::channel::mpsc::unbounded();
+		let (global_tx, global_rx) = tracing_unbounded("test");
 
 		let until_imported = UntilGlobalMessageBlocksImported::new(
 			import_notifications,
@@ -929,7 +929,7 @@ mod tests {
 		let (chain_state, import_notifications) = TestChainState::new();
 		let block_status = chain_state.block_status();
 
-		let (global_tx, global_rx) = futures::channel::mpsc::unbounded();
+		let (global_tx, global_rx) = tracing_unbounded("test");
 
 		let block_sync_requester = TestBlockSyncRequester::default();