Additional Metrics collected and exposed via prometheus (#5414)

This PR refactors the metrics measuring and Prometheus exposing entity in sc-service into its own submodule and extends the parameters it exposes by:

- system load average (over one, five and 15min)
- the TCP connection state of the process (lsof), refs #5304
- number of tokio threads
- number of known forks
- counter for items in each unbounded queue (with internal unbounded channels)
- number of file descriptors opened by this process (*nix only at this point)
- number of system threads (*nix only at this point)

refs #4679

Co-authored-by: Max Inden <mail@max-inden.de>
Co-authored-by: Ashley <ashley.ruglys@gmail.com>

substrate/client/transaction-pool/graph/Cargo.toml

@@ -16,6 +16,7 @@ parking_lot = "0.10.0"
 serde = { version = "1.0.101", features = ["derive"] }
 wasm-timer = "0.2"
 sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" }
+sp-utils = { version = "2.0.0-alpha.5", path = "../../../primitives/utils" }
 sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" }
 sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" }
 sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../../primitives/transaction-pool" }

substrate/client/transaction-pool/graph/src/pool.rs

@@ -24,10 +24,7 @@ use crate::base_pool as base;
 use crate::watcher::Watcher;
 use serde::Serialize;
 
-use futures::{
-	Future, FutureExt,
-	channel::mpsc,
-};
+use futures::{Future, FutureExt};
 use sp_runtime::{
 	generic::BlockId,
 	traits::{self, SaturatedConversion},
@@ -37,12 +34,13 @@ use sp_runtime::{
 };
 use sp_transaction_pool::error;
 use wasm_timer::Instant;
+use sp_utils::mpsc::TracingUnboundedReceiver;
 
 use crate::validated_pool::ValidatedPool;
 pub use crate::validated_pool::ValidatedTransaction;
 
 /// Modification notification event stream type;
-pub type EventStream<H> = mpsc::UnboundedReceiver<H>;
+pub type EventStream<H> = TracingUnboundedReceiver<H>;
 
 /// Extrinsic hash type for a pool.
 pub type ExHash<A> = <A as ChainApi>::Hash;

substrate/client/transaction-pool/graph/src/validated_pool.rs

@@ -27,7 +27,6 @@ use crate::watcher::Watcher;
 use serde::Serialize;
 use log::{debug, warn};
 
-use futures::channel::mpsc;
 use parking_lot::{Mutex, RwLock};
 use sp_runtime::{
 	generic::BlockId,
@@ -36,6 +35,7 @@ use sp_runtime::{
 };
 use sp_transaction_pool::{error, PoolStatus};
 use wasm_timer::Instant;
+use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender};
 
 use crate::base_pool::PruneStatus;
 use crate::pool::{EventStream, Options, ChainApi, ExHash, ExtrinsicFor, TransactionFor};
@@ -95,7 +95,7 @@ pub struct ValidatedPool<B: ChainApi> {
 		ExHash<B>,
 		ExtrinsicFor<B>,
 	>>,
-	import_notification_sinks: Mutex<Vec<mpsc::UnboundedSender<ExHash<B>>>>,
+	import_notification_sinks: Mutex<Vec<TracingUnboundedSender<ExHash<B>>>>,
 	rotator: PoolRotator<ExHash<B>>,
 }
 
@@ -504,7 +504,7 @@ impl<B: ChainApi> ValidatedPool<B> {
 	/// Consumers of this stream should use the `ready` method to actually get the
 	/// pending transactions in the right order.
 	pub fn import_notification_stream(&self) -> EventStream<ExHash<B>> {
-		let (sink, stream) = mpsc::unbounded();
+		let (sink, stream) = tracing_unbounded("mpsc_import_notifications");
 		self.import_notification_sinks.lock().push(sink);
 		stream
 	}

substrate/client/transaction-pool/graph/src/watcher.rs

@@ -16,18 +16,16 @@
 
 //! Extrinsics status updates.
 
-use futures::{
-	Stream,
-	channel::mpsc,
-};
+use futures::Stream;
 use sp_transaction_pool::TransactionStatus;
+use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver};
 
 /// Extrinsic watcher.
 ///
 /// Represents a stream of status updates for particular extrinsic.
 #[derive(Debug)]
 pub struct Watcher<H, BH> {
-	receiver: mpsc::UnboundedReceiver<TransactionStatus<H, BH>>,
+	receiver: TracingUnboundedReceiver<TransactionStatus<H, BH>>,
 	hash: H,
 }
 
@@ -48,7 +46,7 @@ impl<H, BH> Watcher<H, BH> {
 /// Sender part of the watcher. Exposed only for testing purposes.
 #[derive(Debug)]
 pub struct Sender<H, BH> {
-	receivers: Vec<mpsc::UnboundedSender<TransactionStatus<H, BH>>>,
+	receivers: Vec<TracingUnboundedSender<TransactionStatus<H, BH>>>,
 	is_finalized: bool,
 }
 
@@ -64,7 +62,7 @@ impl<H, BH> Default for Sender<H, BH> {
 impl<H: Clone, BH: Clone> Sender<H, BH> {
 	/// Add a new watcher to this sender object.
 	pub fn new_watcher(&mut self, hash: H) -> Watcher<H, BH> {
-		let (tx, receiver) = mpsc::unbounded();
+		let (tx, receiver) = tracing_unbounded("mpsc_txpool_watcher");
 		self.receivers.push(tx);
 		Watcher {
 			receiver,