Merge commit 'e5bed7ac380b6adb54b60a2a72a2a8f07f50d6c1' as 'bridges'

polkadot/bridges/relays/utils/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "relay-utils"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+
+[dependencies]
+ansi_term = "0.12"
+async-std = "1.6.5"
+async-trait = "0.1.40"
+backoff = "0.2"
+isahc = "1.2"
+env_logger = "0.8.2"
+futures = "0.3.5"
+jsonpath_lib = "0.2"
+log = "0.4.11"
+num-traits = "0.2"
+serde_json = "1.0"
+sysinfo = "0.15"
+time = "0.2"
+
+# Substrate dependencies
+
+substrate-prometheus-endpoint = { git = "https://github.com/paritytech/substrate", branch = "master" }

polkadot/bridges/relays/utils/src/initialize.rs

@@ -0,0 +1,95 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Relayer initialization functions.
+
+use std::{fmt::Display, io::Write};
+
+/// Initialize relay environment.
+pub fn initialize_relay() {
+	initialize_logger(true);
+}
+
+/// Initialize Relay logger instance.
+pub fn initialize_logger(with_timestamp: bool) {
+	let mut builder = env_logger::Builder::new();
+	builder.filter_level(log::LevelFilter::Warn);
+	builder.filter_module("bridge", log::LevelFilter::Info);
+	builder.parse_default_env();
+	if with_timestamp {
+		builder.format(move |buf, record| {
+			let timestamp = time::OffsetDateTime::try_now_local()
+				.unwrap_or_else(|_| time::OffsetDateTime::now_utc())
+				.format("%Y-%m-%d %H:%M:%S %z");
+
+			let log_level = color_level(record.level());
+			let log_target = color_target(record.target());
+			let timestamp = if cfg!(windows) {
+				Either::Left(timestamp)
+			} else {
+				Either::Right(ansi_term::Colour::Fixed(8).bold().paint(timestamp))
+			};
+
+			writeln!(buf, "{} {} {} {}", timestamp, log_level, log_target, record.args(),)
+		});
+	} else {
+		builder.format(move |buf, record| {
+			let log_level = color_level(record.level());
+			let log_target = color_target(record.target());
+
+			writeln!(buf, "{} {} {}", log_level, log_target, record.args(),)
+		});
+	}
+
+	builder.init();
+}
+
+enum Either<A, B> {
+	Left(A),
+	Right(B),
+}
+impl<A: Display, B: Display> Display for Either<A, B> {
+	fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+		match self {
+			Self::Left(a) => write!(fmt, "{}", a),
+			Self::Right(b) => write!(fmt, "{}", b),
+		}
+	}
+}
+
+fn color_target(target: &str) -> impl Display + '_ {
+	if cfg!(windows) {
+		Either::Left(target)
+	} else {
+		Either::Right(ansi_term::Colour::Fixed(8).paint(target))
+	}
+}
+
+fn color_level(level: log::Level) -> impl Display {
+	if cfg!(windows) {
+		Either::Left(level)
+	} else {
+		let s = level.to_string();
+		use ansi_term::Colour as Color;
+		Either::Right(match level {
+			log::Level::Error => Color::Fixed(9).bold().paint(s),
+			log::Level::Warn => Color::Fixed(11).bold().paint(s),
+			log::Level::Info => Color::Fixed(10).paint(s),
+			log::Level::Debug => Color::Fixed(14).paint(s),
+			log::Level::Trace => Color::Fixed(12).paint(s),
+		})
+	}
+}

polkadot/bridges/relays/utils/src/lib.rs

@@ -0,0 +1,277 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Utilities used by different relays.
+
+pub use relay_loop::{relay_loop, relay_metrics};
+
+use backoff::{backoff::Backoff, ExponentialBackoff};
+use futures::future::FutureExt;
+use std::time::Duration;
+
+/// Max delay after connection-unrelated error happened before we'll try the
+/// same request again.
+pub const MAX_BACKOFF_INTERVAL: Duration = Duration::from_secs(60);
+/// Delay after connection-related error happened before we'll try
+/// reconnection again.
+pub const CONNECTION_ERROR_DELAY: Duration = Duration::from_secs(10);
+
+pub mod initialize;
+pub mod metrics;
+pub mod relay_loop;
+
+/// Block number traits shared by all chains that relay is able to serve.
+pub trait BlockNumberBase:
+	'static
+	+ From<u32>
+	+ Into<u64>
+	+ Ord
+	+ Clone
+	+ Copy
+	+ Default
+	+ Send
+	+ Sync
+	+ std::fmt::Debug
+	+ std::fmt::Display
+	+ std::hash::Hash
+	+ std::ops::Add<Output = Self>
+	+ std::ops::Sub<Output = Self>
+	+ num_traits::CheckedSub
+	+ num_traits::Saturating
+	+ num_traits::Zero
+	+ num_traits::One
+{
+}
+
+impl<T> BlockNumberBase for T where
+	T: 'static
+		+ From<u32>
+		+ Into<u64>
+		+ Ord
+		+ Clone
+		+ Copy
+		+ Default
+		+ Send
+		+ Sync
+		+ std::fmt::Debug
+		+ std::fmt::Display
+		+ std::hash::Hash
+		+ std::ops::Add<Output = Self>
+		+ std::ops::Sub<Output = Self>
+		+ num_traits::CheckedSub
+		+ num_traits::Saturating
+		+ num_traits::Zero
+		+ num_traits::One
+{
+}
+
+/// Macro that returns (client, Err(error)) tuple from function if result is Err(error).
+#[macro_export]
+macro_rules! bail_on_error {
+	($result: expr) => {
+		match $result {
+			(client, Ok(result)) => (client, result),
+			(client, Err(error)) => return (client, Err(error)),
+		}
+	};
+}
+
+/// Macro that returns (client, Err(error)) tuple from function if result is Err(error).
+#[macro_export]
+macro_rules! bail_on_arg_error {
+	($result: expr, $client: ident) => {
+		match $result {
+			Ok(result) => result,
+			Err(error) => return ($client, Err(error)),
+		}
+	};
+}
+
+/// Ethereum header Id.
+#[derive(Debug, Default, Clone, Copy, Eq, Hash, PartialEq)]
+pub struct HeaderId<Hash, Number>(pub Number, pub Hash);
+
+/// Error type that can signal connection errors.
+pub trait MaybeConnectionError {
+	/// Returns true if error (maybe) represents connection error.
+	fn is_connection_error(&self) -> bool;
+}
+
+/// Stringified error that may be either connection-related or not.
+#[derive(Debug)]
+pub enum StringifiedMaybeConnectionError {
+	/// The error is connection-related error.
+	Connection(String),
+	/// The error is connection-unrelated error.
+	NonConnection(String),
+}
+
+impl StringifiedMaybeConnectionError {
+	/// Create new stringified connection error.
+	pub fn new(is_connection_error: bool, error: String) -> Self {
+		if is_connection_error {
+			StringifiedMaybeConnectionError::Connection(error)
+		} else {
+			StringifiedMaybeConnectionError::NonConnection(error)
+		}
+	}
+}
+
+impl MaybeConnectionError for StringifiedMaybeConnectionError {
+	fn is_connection_error(&self) -> bool {
+		match *self {
+			StringifiedMaybeConnectionError::Connection(_) => true,
+			StringifiedMaybeConnectionError::NonConnection(_) => false,
+		}
+	}
+}
+
+impl ToString for StringifiedMaybeConnectionError {
+	fn to_string(&self) -> String {
+		match *self {
+			StringifiedMaybeConnectionError::Connection(ref err) => err.clone(),
+			StringifiedMaybeConnectionError::NonConnection(ref err) => err.clone(),
+		}
+	}
+}
+
+/// Exponential backoff for connection-unrelated errors retries.
+pub fn retry_backoff() -> ExponentialBackoff {
+	ExponentialBackoff {
+		// we do not want relayer to stop
+		max_elapsed_time: None,
+		max_interval: MAX_BACKOFF_INTERVAL,
+		..Default::default()
+	}
+}
+
+/// Compact format of IDs vector.
+pub fn format_ids<Id: std::fmt::Debug>(mut ids: impl ExactSizeIterator<Item = Id>) -> String {
+	const NTH_PROOF: &str = "we have checked len; qed";
+	match ids.len() {
+		0 => "<nothing>".into(),
+		1 => format!("{:?}", ids.next().expect(NTH_PROOF)),
+		2 => {
+			let id0 = ids.next().expect(NTH_PROOF);
+			let id1 = ids.next().expect(NTH_PROOF);
+			format!("[{:?}, {:?}]", id0, id1)
+		}
+		len => {
+			let id0 = ids.next().expect(NTH_PROOF);
+			let id_last = ids.last().expect(NTH_PROOF);
+			format!("{}:[{:?} ... {:?}]", len, id0, id_last)
+		}
+	}
+}
+
+/// Stream that emits item every `timeout_ms` milliseconds.
+pub fn interval(timeout: Duration) -> impl futures::Stream<Item = ()> {
+	futures::stream::unfold((), move |_| async move {
+		async_std::task::sleep(timeout).await;
+		Some(((), ()))
+	})
+}
+
+/// Which client has caused error.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum FailedClient {
+	/// It is the source client who has caused error.
+	Source,
+	/// It is the target client who has caused error.
+	Target,
+	/// Both clients are failing, or we just encountered some other error that
+	/// should be treated like that.
+	Both,
+}
+
+/// Future process result.
+#[derive(Debug, Clone, Copy)]
+pub enum ProcessFutureResult {
+	/// Future has been processed successfully.
+	Success,
+	/// Future has failed with non-connection error.
+	Failed,
+	/// Future has failed with connection error.
+	ConnectionFailed,
+}
+
+impl ProcessFutureResult {
+	/// Returns true if result is Success.
+	pub fn is_ok(self) -> bool {
+		match self {
+			ProcessFutureResult::Success => true,
+			ProcessFutureResult::Failed | ProcessFutureResult::ConnectionFailed => false,
+		}
+	}
+
+	/// Returns Ok(true) if future has succeeded.
+	/// Returns Ok(false) if future has failed with non-connection error.
+	/// Returns Err if future is `ConnectionFailed`.
+	pub fn fail_if_connection_error(self, failed_client: FailedClient) -> Result<bool, FailedClient> {
+		match self {
+			ProcessFutureResult::Success => Ok(true),
+			ProcessFutureResult::Failed => Ok(false),
+			ProcessFutureResult::ConnectionFailed => Err(failed_client),
+		}
+	}
+}
+
+/// Process result of the future from a client.
+pub fn process_future_result<TResult, TError, TGoOfflineFuture>(
+	result: Result<TResult, TError>,
+	retry_backoff: &mut ExponentialBackoff,
+	on_success: impl FnOnce(TResult),
+	go_offline_future: &mut std::pin::Pin<&mut futures::future::Fuse<TGoOfflineFuture>>,
+	go_offline: impl FnOnce(Duration) -> TGoOfflineFuture,
+	error_pattern: impl FnOnce() -> String,
+) -> ProcessFutureResult
+where
+	TError: std::fmt::Debug + MaybeConnectionError,
+	TGoOfflineFuture: FutureExt,
+{
+	match result {
+		Ok(result) => {
+			on_success(result);
+			retry_backoff.reset();
+			ProcessFutureResult::Success
+		}
+		Err(error) if error.is_connection_error() => {
+			log::error!(
+				target: "bridge",
+				"{}: {:?}. Going to restart",
+				error_pattern(),
+				error,
+			);
+
+			retry_backoff.reset();
+			go_offline_future.set(go_offline(CONNECTION_ERROR_DELAY).fuse());
+			ProcessFutureResult::ConnectionFailed
+		}
+		Err(error) => {
+			let retry_delay = retry_backoff.next_backoff().unwrap_or(CONNECTION_ERROR_DELAY);
+			log::error!(
+				target: "bridge",
+				"{}: {:?}. Retrying in {}",
+				error_pattern(),
+				error,
+				retry_delay.as_secs_f64(),
+			);
+
+			go_offline_future.set(go_offline(retry_delay).fuse());
+			ProcessFutureResult::Failed
+		}
+	}
+}

polkadot/bridges/relays/utils/src/metrics.rs

@@ -0,0 +1,162 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+pub use float_json_value::FloatJsonValueMetric;
+pub use global::GlobalMetrics;
+pub use substrate_prometheus_endpoint::{
+	prometheus::core::{Atomic, Collector},
+	register, Counter, CounterVec, Gauge, GaugeVec, Opts, PrometheusError, Registry, F64, U64,
+};
+
+use async_trait::async_trait;
+use std::{fmt::Debug, time::Duration};
+
+mod float_json_value;
+mod global;
+
+/// Unparsed address that needs to be used to expose Prometheus metrics.
+#[derive(Debug, Clone)]
+pub struct MetricsAddress {
+	/// Serve HTTP requests at given host.
+	pub host: String,
+	/// Serve HTTP requests at given port.
+	pub port: u16,
+}
+
+/// Prometheus endpoint MetricsParams.
+#[derive(Debug, Clone)]
+pub struct MetricsParams {
+	/// Interface and TCP port to be used when exposing Prometheus metrics.
+	pub address: Option<MetricsAddress>,
+	/// Metrics registry. May be `Some(_)` if several components share the same endpoint.
+	pub registry: Option<Registry>,
+	/// Prefix that must be used in metric names.
+	pub metrics_prefix: Option<String>,
+}
+
+/// Metrics API.
+pub trait Metrics: Clone + Send + Sync + 'static {}
+
+impl<T: Clone + Send + Sync + 'static> Metrics for T {}
+
+/// Standalone metrics API.
+///
+/// Metrics of this kind know how to update themselves, so we may just spawn and forget the
+/// asynchronous self-update task.
+#[async_trait]
+pub trait StandaloneMetrics: Metrics {
+	/// Update metric values.
+	async fn update(&self);
+
+	/// Metrics update interval.
+	fn update_interval(&self) -> Duration;
+
+	/// Spawn the self update task that will keep update metric value at given intervals.
+	fn spawn(self) {
+		async_std::task::spawn(async move {
+			let update_interval = self.update_interval();
+			loop {
+				self.update().await;
+				async_std::task::sleep(update_interval).await;
+			}
+		});
+	}
+}
+
+impl Default for MetricsAddress {
+	fn default() -> Self {
+		MetricsAddress {
+			host: "127.0.0.1".into(),
+			port: 9616,
+		}
+	}
+}
+
+impl MetricsParams {
+	/// Creates metrics params so that metrics are not exposed.
+	pub fn disabled() -> Self {
+		MetricsParams {
+			address: None,
+			registry: None,
+			metrics_prefix: None,
+		}
+	}
+
+	/// Do not expose metrics.
+	pub fn disable(mut self) -> Self {
+		self.address = None;
+		self
+	}
+
+	/// Set prefix to use in metric names.
+	pub fn metrics_prefix(mut self, prefix: String) -> Self {
+		self.metrics_prefix = Some(prefix);
+		self
+	}
+}
+
+impl From<Option<MetricsAddress>> for MetricsParams {
+	fn from(address: Option<MetricsAddress>) -> Self {
+		MetricsParams {
+			address,
+			registry: None,
+			metrics_prefix: None,
+		}
+	}
+}
+
+/// Returns metric name optionally prefixed with given prefix.
+pub fn metric_name(prefix: Option<&str>, name: &str) -> String {
+	if let Some(prefix) = prefix {
+		format!("{}_{}", prefix, name)
+	} else {
+		name.into()
+	}
+}
+
+/// Set value of gauge metric.
+///
+/// If value is `Ok(None)` or `Err(_)`, metric would have default value.
+pub fn set_gauge_value<T: Default + Debug, V: Atomic<T = T>, E: Debug>(gauge: &Gauge<V>, value: Result<Option<T>, E>) {
+	gauge.set(match value {
+		Ok(Some(value)) => {
+			log::trace!(
+				target: "bridge-metrics",
+				"Updated value of metric '{:?}': {:?}",
+				gauge.desc().first().map(|d| &d.fq_name),
+				value,
+			);
+			value
+		}
+		Ok(None) => {
+			log::warn!(
+				target: "bridge-metrics",
+				"Failed to update metric '{:?}': value is empty",
+				gauge.desc().first().map(|d| &d.fq_name),
+			);
+			Default::default()
+		}
+		Err(error) => {
+			log::warn!(
+				target: "bridge-metrics",
+				"Failed to update metric '{:?}': {:?}",
+				gauge.desc().first().map(|d| &d.fq_name),
+				error,
+			);
+			Default::default()
+		}
+	})
+}

polkadot/bridges/relays/utils/src/metrics/float_json_value.rs

@@ -0,0 +1,121 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::metrics::{metric_name, register, Gauge, PrometheusError, Registry, StandaloneMetrics, F64};
+
+use async_trait::async_trait;
+use std::time::Duration;
+
+/// Value update interval.
+const UPDATE_INTERVAL: Duration = Duration::from_secs(60);
+
+/// Metric that represents float value received from HTTP service as float gauge.
+#[derive(Debug, Clone)]
+pub struct FloatJsonValueMetric {
+	url: String,
+	json_path: String,
+	metric: Gauge<F64>,
+}
+
+impl FloatJsonValueMetric {
+	/// Create new metric instance with given name and help.
+	pub fn new(
+		registry: &Registry,
+		prefix: Option<&str>,
+		url: String,
+		json_path: String,
+		name: String,
+		help: String,
+	) -> Result<Self, PrometheusError> {
+		Ok(FloatJsonValueMetric {
+			url,
+			json_path,
+			metric: register(Gauge::new(metric_name(prefix, &name), help)?, registry)?,
+		})
+	}
+
+	/// Read value from HTTP service.
+	async fn read_value(&self) -> Result<f64, String> {
+		use isahc::{AsyncReadResponseExt, HttpClient, Request};
+
+		fn map_isahc_err(err: impl std::fmt::Display) -> String {
+			format!("Failed to fetch token price from remote server: {}", err)
+		}
+
+		let request = Request::get(&self.url)
+			.header("Accept", "application/json")
+			.body(())
+			.map_err(map_isahc_err)?;
+		let raw_response = HttpClient::new()
+			.map_err(map_isahc_err)?
+			.send_async(request)
+			.await
+			.map_err(map_isahc_err)?
+			.text()
+			.await
+			.map_err(map_isahc_err)?;
+
+		parse_service_response(&self.json_path, &raw_response)
+	}
+}
+
+#[async_trait]
+impl StandaloneMetrics for FloatJsonValueMetric {
+	fn update_interval(&self) -> Duration {
+		UPDATE_INTERVAL
+	}
+
+	async fn update(&self) {
+		crate::metrics::set_gauge_value(&self.metric, self.read_value().await.map(Some));
+	}
+}
+
+/// Parse HTTP service response.
+fn parse_service_response(json_path: &str, response: &str) -> Result<f64, String> {
+	let json = serde_json::from_str(response).map_err(|err| {
+		format!(
+			"Failed to parse HTTP service response: {:?}. Response: {:?}",
+			err, response,
+		)
+	})?;
+
+	let mut selector = jsonpath_lib::selector(&json);
+	let maybe_selected_value = selector(json_path).map_err(|err| {
+		format!(
+			"Failed to select value from response: {:?}. Response: {:?}",
+			err, response,
+		)
+	})?;
+	let selected_value = maybe_selected_value
+		.first()
+		.and_then(|v| v.as_f64())
+		.ok_or_else(|| format!("Missing required value from response: {:?}", response,))?;
+
+	Ok(selected_value)
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn parse_service_response_works() {
+		assert_eq!(
+			parse_service_response("$.kusama.usd", r#"{"kusama":{"usd":433.05}}"#).map_err(drop),
+			Ok(433.05),
+		);
+	}
+}

polkadot/bridges/relays/utils/src/metrics/global.rs

@@ -0,0 +1,111 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Global system-wide Prometheus metrics exposed by relays.
+
+use crate::metrics::{
+	metric_name, register, Gauge, GaugeVec, Opts, PrometheusError, Registry, StandaloneMetrics, F64, U64,
+};
+
+use async_std::sync::{Arc, Mutex};
+use async_trait::async_trait;
+use std::time::Duration;
+use sysinfo::{ProcessExt, RefreshKind, System, SystemExt};
+
+/// Global metrics update interval.
+const UPDATE_INTERVAL: Duration = Duration::from_secs(10);
+
+/// Global Prometheus metrics.
+#[derive(Debug, Clone)]
+pub struct GlobalMetrics {
+	system: Arc<Mutex<System>>,
+	system_average_load: GaugeVec<F64>,
+	process_cpu_usage_percentage: Gauge<F64>,
+	process_memory_usage_bytes: Gauge<U64>,
+}
+
+impl GlobalMetrics {
+	/// Create and register global metrics.
+	pub fn new(registry: &Registry, prefix: Option<&str>) -> Result<Self, PrometheusError> {
+		Ok(GlobalMetrics {
+			system: Arc::new(Mutex::new(System::new_with_specifics(RefreshKind::everything()))),
+			system_average_load: register(
+				GaugeVec::new(
+					Opts::new(metric_name(prefix, "system_average_load"), "System load average"),
+					&["over"],
+				)?,
+				registry,
+			)?,
+			process_cpu_usage_percentage: register(
+				Gauge::new(metric_name(prefix, "process_cpu_usage_percentage"), "Process CPU usage")?,
+				registry,
+			)?,
+			process_memory_usage_bytes: register(
+				Gauge::new(
+					metric_name(prefix, "process_memory_usage_bytes"),
+					"Process memory (resident set size) usage",
+				)?,
+				registry,
+			)?,
+		})
+	}
+}
+
+#[async_trait]
+impl StandaloneMetrics for GlobalMetrics {
+	async fn update(&self) {
+		// update system-wide metrics
+		let mut system = self.system.lock().await;
+		let load = system.get_load_average();
+		self.system_average_load.with_label_values(&["1min"]).set(load.one);
+		self.system_average_load.with_label_values(&["5min"]).set(load.five);
+		self.system_average_load.with_label_values(&["15min"]).set(load.fifteen);
+
+		// update process-related metrics
+		let pid = sysinfo::get_current_pid().expect(
+			"only fails where pid is unavailable (os=unknown || arch=wasm32);\
+				relay is not supposed to run in such MetricsParamss;\
+				qed",
+		);
+		let is_process_refreshed = system.refresh_process(pid);
+		match (is_process_refreshed, system.get_process(pid)) {
+			(true, Some(process_info)) => {
+				let cpu_usage = process_info.cpu_usage() as f64;
+				let memory_usage = process_info.memory() * 1024;
+				log::trace!(
+					target: "bridge-metrics",
+					"Refreshed process metrics: CPU={}, memory={}",
+					cpu_usage,
+					memory_usage,
+				);
+
+				self.process_cpu_usage_percentage
+					.set(if cpu_usage.is_finite() { cpu_usage } else { 0f64 });
+				self.process_memory_usage_bytes.set(memory_usage);
+			}
+			_ => {
+				log::warn!(
+					target: "bridge-metrics",
+					"Failed to refresh process information. Metrics may show obsolete values",
+				);
+			}
+		}
+	}
+
+	fn update_interval(&self) -> Duration {
+		UPDATE_INTERVAL
+	}
+}

polkadot/bridges/relays/utils/src/relay_loop.rs

@@ -0,0 +1,256 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::metrics::{Metrics, MetricsAddress, MetricsParams, PrometheusError, StandaloneMetrics};
+use crate::{FailedClient, MaybeConnectionError};
+
+use async_trait::async_trait;
+use std::{fmt::Debug, future::Future, net::SocketAddr, time::Duration};
+use substrate_prometheus_endpoint::{init_prometheus, Registry};
+
+/// Default pause between reconnect attempts.
+pub const RECONNECT_DELAY: Duration = Duration::from_secs(10);
+
+/// Basic blockchain client from relay perspective.
+#[async_trait]
+pub trait Client: Clone + Send + Sync {
+	/// Type of error this clients returns.
+	type Error: Debug + MaybeConnectionError;
+
+	/// Try to reconnect to source node.
+	async fn reconnect(&mut self) -> Result<(), Self::Error>;
+}
+
+/// Returns generic loop that may be customized and started.
+pub fn relay_loop<SC, TC>(source_client: SC, target_client: TC) -> Loop<SC, TC, ()> {
+	Loop {
+		reconnect_delay: RECONNECT_DELAY,
+		source_client,
+		target_client,
+		loop_metric: None,
+	}
+}
+
+/// Returns generic relay loop metrics that may be customized and used in one or several relay loops.
+pub fn relay_metrics(prefix: Option<String>, params: MetricsParams) -> LoopMetrics<(), (), ()> {
+	LoopMetrics {
+		relay_loop: Loop {
+			reconnect_delay: RECONNECT_DELAY,
+			source_client: (),
+			target_client: (),
+			loop_metric: None,
+		},
+		address: params.address,
+		registry: params.registry.unwrap_or_else(|| create_metrics_registry(prefix)),
+		metrics_prefix: params.metrics_prefix,
+		loop_metric: None,
+	}
+}
+
+/// Generic relay loop.
+pub struct Loop<SC, TC, LM> {
+	reconnect_delay: Duration,
+	source_client: SC,
+	target_client: TC,
+	loop_metric: Option<LM>,
+}
+
+/// Relay loop metrics builder.
+pub struct LoopMetrics<SC, TC, LM> {
+	relay_loop: Loop<SC, TC, ()>,
+	address: Option<MetricsAddress>,
+	registry: Registry,
+	metrics_prefix: Option<String>,
+	loop_metric: Option<LM>,
+}
+
+impl<SC, TC, LM> Loop<SC, TC, LM> {
+	/// Customize delay between reconnect attempts.
+	pub fn reconnect_delay(mut self, reconnect_delay: Duration) -> Self {
+		self.reconnect_delay = reconnect_delay;
+		self
+	}
+
+	/// Start building loop metrics using given prefix.
+	pub fn with_metrics(self, prefix: Option<String>, params: MetricsParams) -> LoopMetrics<SC, TC, ()> {
+		LoopMetrics {
+			relay_loop: Loop {
+				reconnect_delay: self.reconnect_delay,
+				source_client: self.source_client,
+				target_client: self.target_client,
+				loop_metric: None,
+			},
+			address: params.address,
+			registry: params.registry.unwrap_or_else(|| create_metrics_registry(prefix)),
+			metrics_prefix: params.metrics_prefix,
+			loop_metric: None,
+		}
+	}
+
+	/// Run relay loop.
+	///
+	/// This function represents an outer loop, which in turn calls provided `run_loop` function to do
+	/// actual job. When `run_loop` returns, this outer loop reconnects to failed client (source,
+	/// target or both) and calls `run_loop` again.
+	pub async fn run<R, F>(mut self, run_loop: R) -> Result<(), String>
+	where
+		R: Fn(SC, TC, Option<LM>) -> F,
+		F: Future<Output = Result<(), FailedClient>>,
+		SC: Client,
+		TC: Client,
+		LM: Clone,
+	{
+		loop {
+			let result = run_loop(
+				self.source_client.clone(),
+				self.target_client.clone(),
+				self.loop_metric.clone(),
+			)
+			.await;
+
+			match result {
+				Ok(()) => break,
+				Err(failed_client) => loop {
+					async_std::task::sleep(self.reconnect_delay).await;
+					if failed_client == FailedClient::Both || failed_client == FailedClient::Source {
+						match self.source_client.reconnect().await {
+							Ok(()) => (),
+							Err(error) => {
+								log::warn!(
+									target: "bridge",
+									"Failed to reconnect to source client. Going to retry in {}s: {:?}",
+									self.reconnect_delay.as_secs(),
+									error,
+								);
+								continue;
+							}
+						}
+					}
+					if failed_client == FailedClient::Both || failed_client == FailedClient::Target {
+						match self.target_client.reconnect().await {
+							Ok(()) => (),
+							Err(error) => {
+								log::warn!(
+									target: "bridge",
+									"Failed to reconnect to target client. Going to retry in {}s: {:?}",
+									self.reconnect_delay.as_secs(),
+									error,
+								);
+								continue;
+							}
+						}
+					}
+
+					break;
+				},
+			}
+
+			log::debug!(target: "bridge", "Restarting relay loop");
+		}
+
+		Ok(())
+	}
+}
+
+impl<SC, TC, LM> LoopMetrics<SC, TC, LM> {
+	/// Add relay loop metrics.
+	///
+	/// Loop metrics will be passed to the loop callback.
+	pub fn loop_metric<NewLM: Metrics>(
+		self,
+		create_metric: impl FnOnce(&Registry, Option<&str>) -> Result<NewLM, PrometheusError>,
+	) -> Result<LoopMetrics<SC, TC, NewLM>, String> {
+		let loop_metric = create_metric(&self.registry, self.metrics_prefix.as_deref()).map_err(|e| e.to_string())?;
+
+		Ok(LoopMetrics {
+			relay_loop: self.relay_loop,
+			address: self.address,
+			registry: self.registry,
+			metrics_prefix: self.metrics_prefix,
+			loop_metric: Some(loop_metric),
+		})
+	}
+
+	/// Add standalone metrics.
+	pub fn standalone_metric<M: StandaloneMetrics>(
+		self,
+		create_metric: impl FnOnce(&Registry, Option<&str>) -> Result<M, PrometheusError>,
+	) -> Result<Self, String> {
+		// since standalone metrics are updating themselves, we may just ignore the fact that the same
+		// standalone metric is exposed by several loops && only spawn single metric
+		match create_metric(&self.registry, self.metrics_prefix.as_deref()) {
+			Ok(standalone_metrics) => standalone_metrics.spawn(),
+			Err(PrometheusError::AlreadyReg) => (),
+			Err(e) => return Err(e.to_string()),
+		}
+
+		Ok(self)
+	}
+
+	/// Convert into `MetricsParams` structure so that metrics registry may be extended later.
+	pub fn into_params(self) -> MetricsParams {
+		MetricsParams {
+			address: self.address,
+			registry: Some(self.registry),
+			metrics_prefix: self.metrics_prefix,
+		}
+	}
+
+	/// Expose metrics using address passed at creation.
+	///
+	/// If passed `address` is `None`, metrics are not exposed.
+	pub async fn expose(self) -> Result<Loop<SC, TC, LM>, String> {
+		if let Some(address) = self.address {
+			let socket_addr = SocketAddr::new(
+				address.host.parse().map_err(|err| {
+					format!(
+						"Invalid host {} is used to expose Prometheus metrics: {}",
+						address.host, err,
+					)
+				})?,
+				address.port,
+			);
+
+			let registry = self.registry;
+			async_std::task::spawn(async move {
+				let result = init_prometheus(socket_addr, registry).await;
+				log::trace!(
+					target: "bridge-metrics",
+					"Prometheus endpoint has exited with result: {:?}",
+					result,
+				);
+			});
+		}
+
+		Ok(Loop {
+			reconnect_delay: self.relay_loop.reconnect_delay,
+			source_client: self.relay_loop.source_client,
+			target_client: self.relay_loop.target_client,
+			loop_metric: self.loop_metric,
+		})
+	}
+}
+
+/// Create new registry with global metrics.
+fn create_metrics_registry(prefix: Option<String>) -> Registry {
+	match prefix {
+		Some(prefix) => {
+			assert!(!prefix.is_empty(), "Metrics prefix can not be empty");
+			Registry::new_custom(Some(prefix), None).expect("only fails if prefix is empty; prefix is not empty; qed")
+		}
+		None => Registry::new(),
+	}
+}