Additional Metrics collected and exposed via prometheus (#5414)

This PR refactors the metrics measuring and Prometheus exposing entity in sc-service into its own submodule and extends the parameters it exposes by:

- system load average (over one, five and 15min)
- the TCP connection state of the process (lsof), refs #5304
- number of tokio threads
- number of known forks
- counter for items in each unbounded queue (with internal unbounded channels)
- number of file descriptors opened by this process (*nix only at this point)
- number of system threads (*nix only at this point)

refs #4679

Co-authored-by: Max Inden <mail@max-inden.de>
Co-authored-by: Ashley <ashley.ruglys@gmail.com>

substrate/client/offchain/src/api/http.rs

@@ -32,11 +32,12 @@ use futures::{prelude::*, future, channel::mpsc};
 use log::error;
 use sp_core::offchain::{HttpRequestId, Timestamp, HttpRequestStatus, HttpError};
 use std::{fmt, io::Read as _, mem, pin::Pin, task::Context, task::Poll};
+use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver};
 
 /// Creates a pair of [`HttpApi`] and [`HttpWorker`].
 pub fn http() -> (HttpApi, HttpWorker) {
-	let (to_worker, from_api) = mpsc::unbounded();
-	let (to_api, from_worker) = mpsc::unbounded();
+	let (to_worker, from_api) = tracing_unbounded("mpsc_ocw_to_worker");
+	let (to_api, from_worker) = tracing_unbounded("mpsc_ocw_to_api");
 
 	let api = HttpApi {
 		to_worker,
@@ -63,10 +64,10 @@ pub fn http() -> (HttpApi, HttpWorker) {
 /// to offchain workers.
 pub struct HttpApi {
 	/// Used to sends messages to the worker.
-	to_worker: mpsc::UnboundedSender<ApiToWorker>,
+	to_worker: TracingUnboundedSender<ApiToWorker>,
 	/// Used to receive messages from the worker.
 	/// We use a `Fuse` in order to have an extra protection against panicking.
-	from_worker: stream::Fuse<mpsc::UnboundedReceiver<WorkerToApi>>,
+	from_worker: stream::Fuse<TracingUnboundedReceiver<WorkerToApi>>,
 	/// Id to assign to the next HTTP request that is started.
 	next_id: HttpRequestId,
 	/// List of HTTP requests in preparation or in progress.
@@ -546,9 +547,9 @@ enum WorkerToApi {
 /// Must be continuously polled for the [`HttpApi`] to properly work.
 pub struct HttpWorker {
 	/// Used to sends messages to the `HttpApi`.
-	to_api: mpsc::UnboundedSender<WorkerToApi>,
+	to_api: TracingUnboundedSender<WorkerToApi>,
 	/// Used to receive messages from the `HttpApi`.
-	from_api: mpsc::UnboundedReceiver<ApiToWorker>,
+	from_api: TracingUnboundedReceiver<ApiToWorker>,
 	/// The engine that runs HTTP requests.
 	http_client: hyper::Client<hyper_rustls::HttpsConnector<hyper::client::HttpConnector>, hyper::Body>,
 	/// HTTP requests that are being worked on by the engine.