Additional Metrics collected and exposed via prometheus (#5414)

This PR refactors the metrics measuring and Prometheus exposing entity in sc-service into its own submodule and extends the parameters it exposes by:

- system load average (over one, five and 15min)
- the TCP connection state of the process (lsof), refs #5304
- number of tokio threads
- number of known forks
- counter for items in each unbounded queue (with internal unbounded channels)
- number of file descriptors opened by this process (*nix only at this point)
- number of system threads (*nix only at this point)

refs #4679

Co-authored-by: Max Inden <mail@max-inden.de>
Co-authored-by: Ashley <ashley.ruglys@gmail.com>

substrate/primitives/consensus/common/src/import_queue/basic_queue.rs

@@ -15,10 +15,11 @@
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
 use std::{mem, pin::Pin, time::Duration, marker::PhantomData, sync::Arc};
-use futures::{prelude::*, channel::mpsc, task::Context, task::Poll};
+use futures::{prelude::*, task::Context, task::Poll};
 use futures_timer::Delay;
 use parking_lot::{Mutex, Condvar};
 use sp_runtime::{Justification, traits::{Block as BlockT, Header as HeaderT, NumberFor}};
+use sp_utils::mpsc::{TracingUnboundedSender, tracing_unbounded};
 
 use crate::block_import::BlockOrigin;
 use crate::import_queue::{
@@ -32,7 +33,7 @@ use crate::import_queue::{
 /// task, with plugable verification.
 pub struct BasicQueue<B: BlockT, Transaction> {
 	/// Channel to send messages to the background task.
-	sender: mpsc::UnboundedSender<ToWorkerMsg<B>>,
+	sender: TracingUnboundedSender<ToWorkerMsg<B>>,
 	/// Results coming from the worker task.
 	result_port: BufferedLinkReceiver<B>,
 	/// If it isn't possible to spawn the future in `future_to_spawn` (which is notably the case in
@@ -195,8 +196,8 @@ impl<B: BlockT, Transaction: Send> BlockImportWorker<B, Transaction> {
 		block_import: BoxBlockImport<B, Transaction>,
 		justification_import: Option<BoxJustificationImport<B>>,
 		finality_proof_import: Option<BoxFinalityProofImport<B>>,
-	) -> (impl Future<Output = ()> + Send, mpsc::UnboundedSender<ToWorkerMsg<B>>) {
-		let (sender, mut port) = mpsc::unbounded();
+	) -> (impl Future<Output = ()> + Send, TracingUnboundedSender<ToWorkerMsg<B>>) {
+		let (sender, mut port) = tracing_unbounded("mpsc_block_import_worker");
 
 		let mut worker = BlockImportWorker {
 			result_sender,

substrate/primitives/consensus/common/src/import_queue/buffered_link.rs

@@ -37,8 +37,9 @@
 //! ```
 //!
 
-use futures::{prelude::*, channel::mpsc};
+use futures::prelude::*;
 use sp_runtime::traits::{Block as BlockT, NumberFor};
+use sp_utils::mpsc::{TracingUnboundedSender, TracingUnboundedReceiver, tracing_unbounded};
 use std::{pin::Pin, task::Context, task::Poll};
 use crate::import_queue::{Origin, Link, BlockImportResult, BlockImportError};
 
@@ -46,7 +47,7 @@ use crate::import_queue::{Origin, Link, BlockImportResult, BlockImportError};
 /// can be used to buffer commands, and the receiver can be used to poll said commands and transfer
 /// them to another link.
 pub fn buffered_link<B: BlockT>() -> (BufferedLinkSender<B>, BufferedLinkReceiver<B>) {
-	let (tx, rx) = mpsc::unbounded();
+	let (tx, rx) = tracing_unbounded("mpsc_buffered_link");
 	let tx = BufferedLinkSender { tx };
 	let rx = BufferedLinkReceiver { rx };
 	(tx, rx)
@@ -54,7 +55,7 @@ pub fn buffered_link<B: BlockT>() -> (BufferedLinkSender<B>, BufferedLinkReceive
 
 /// See [`buffered_link`].
 pub struct BufferedLinkSender<B: BlockT> {
-	tx: mpsc::UnboundedSender<BlockImportWorkerMsg<B>>,
+	tx: TracingUnboundedSender<BlockImportWorkerMsg<B>>,
 }
 
 impl<B: BlockT> BufferedLinkSender<B> {
@@ -125,7 +126,7 @@ impl<B: BlockT> Link<B> for BufferedLinkSender<B> {
 
 /// See [`buffered_link`].
 pub struct BufferedLinkReceiver<B: BlockT> {
-	rx: mpsc::UnboundedReceiver<BlockImportWorkerMsg<B>>,
+	rx: TracingUnboundedReceiver<BlockImportWorkerMsg<B>>,
 }
 
 impl<B: BlockT> BufferedLinkReceiver<B> {