fix: Complete snowbridge pezpallet rebrand and critical bug fixes

- snowbridge-pezpallet-* → pezsnowbridge-pezpallet-* (201 refs)
- pallet/ directories → pezpallet/ (4 locations)
- Fixed pezpallet.rs self-include recursion bug
- Fixed sc-chain-spec hardcoded crate name in derive macro
- Reverted .pezpallet_by_name() to .pallet_by_name() (subxt API)
- Added BizinikiwiConfig type alias for zombienet tests
- Deleted obsolete session state files

Verified: pezsnowbridge-pezpallet-*, pezpallet-staking,
pezpallet-staking-async, pezframe-benchmarking-cli all pass cargo check

pezbridges/relays/utils/src/error.rs

@@ -0,0 +1,46 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::net::AddrParseError;
+use thiserror::Error;
+
+/// Result type used by relay utilities.
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Relay utilities errors.
+#[derive(Error, Debug)]
+pub enum Error {
+	/// Failed to request a float value from HTTP service.
+	#[error("Failed to fetch token price from remote server: {0}")]
+	FetchTokenPrice(#[source] anyhow::Error),
+	/// Failed to parse the response from HTTP service.
+	#[error("Failed to parse HTTP service response: {0:?}. Response: {1:?}")]
+	ParseHttp(serde_json::Error, String),
+	/// Failed to select response value from the Json response.
+	#[error("Failed to select value from response: {0:?}. Response: {1:?}")]
+	SelectResponseValue(jsonpath_lib::JsonPathError, String),
+	/// Failed to parse float value from the selected value.
+	#[error(
+		"Failed to parse float value {0:?} from response. It is assumed to be positive and normal"
+	)]
+	ParseFloat(f64),
+	/// Couldn't found value in the JSON response.
+	#[error("Missing required value from response: {0:?}")]
+	MissingResponseValue(String),
+	/// Invalid host address was used for exposing Prometheus metrics.
+	#[error("Invalid host {0} is used to expose Prometheus metrics: {1}")]
+	ExposingMetricsInvalidHost(String, AddrParseError),
+	/// Prometheus error.
+	#[error("{0}")]
+	Prometheus(#[from] prometheus_endpoint::prometheus::Error),
+}

pezbridges/relays/utils/src/initialize.rs

@@ -0,0 +1,72 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Relayer initialization functions.
+
+use parking_lot::Mutex;
+use pezsp_tracing::{
+	tracing::Level,
+	tracing_subscriber::{
+		fmt::{time::OffsetTime, SubscriberBuilder},
+		EnvFilter,
+	},
+};
+use std::cell::RefCell;
+
+/// Relayer version that is provided as metric. Must be set by a binary
+/// (get it with `option_env!("CARGO_PKG_VERSION")` from a binary package code).
+pub static RELAYER_VERSION: Mutex<Option<String>> = Mutex::new(None);
+
+async_std::task_local! {
+	pub(crate) static LOOP_NAME: RefCell<String> = RefCell::new(String::default());
+}
+
+/// Initialize relay environment.
+pub fn initialize_relay() {
+	initialize_logger(true);
+}
+
+/// Initialize Relay logger instance.
+pub fn initialize_logger(with_timestamp: bool) {
+	let format = time::format_description::parse(
+		"[year]-[month]-[day] \
+		[hour repr:24]:[minute]:[second] [offset_hour sign:mandatory]",
+	)
+	.expect("static format string is valid");
+
+	let local_time = OffsetTime::new(
+		time::UtcOffset::current_local_offset().unwrap_or(time::UtcOffset::UTC),
+		format,
+	);
+
+	let env_filter = EnvFilter::builder()
+		.with_default_directive(Level::WARN.into())
+		.with_default_directive("bridge=info".parse().expect("static filter string is valid"))
+		.from_env_lossy();
+
+	let builder = SubscriberBuilder::default().with_env_filter(env_filter);
+
+	if with_timestamp {
+		builder.with_timer(local_time).init();
+	} else {
+		builder.without_time().init();
+	}
+}
+
+/// Initialize relay loop. Must only be called once per every loop task.
+pub(crate) fn initialize_loop(loop_name: String) {
+	LOOP_NAME.with(|g_loop_name| *g_loop_name.borrow_mut() = loop_name);
+}

pezbridges/relays/utils/src/lib.rs

@@ -0,0 +1,318 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Utilities used by different relays.
+
+pub use pezbp_runtime::HeaderId;
+pub use error::Error;
+pub use relay_loop::{relay_loop, relay_metrics};
+pub use pezsp_runtime::traits::{UniqueSaturatedFrom, UniqueSaturatedInto};
+use std::fmt::Debug;
+
+use async_trait::async_trait;
+use backoff::{backoff::Backoff, ExponentialBackoff};
+use futures::future::{BoxFuture, FutureExt};
+use std::time::Duration;
+use thiserror::Error;
+
+/// Default relay loop stall timeout. If transactions generated by relay are immortal, then
+/// this timeout is used.
+///
+/// There are no any strict requirements on block time in Bizinikiwi. But we assume here that all
+/// Bizinikiwi-based chains will be designed to produce relatively fast (compared to the slowest
+/// blockchains) blocks. So 1 hour seems to be a good guess for (even congested) chains to mine
+/// transaction, or remove it from the pool.
+pub const STALL_TIMEOUT: Duration = Duration::from_secs(60 * 60);
+
+/// Max delay after connection-unrelated error happened before we'll try the
+/// same request again.
+pub const MAX_BACKOFF_INTERVAL: Duration = Duration::from_secs(60);
+/// Delay after connection-related error happened before we'll try
+/// reconnection again.
+pub const CONNECTION_ERROR_DELAY: Duration = Duration::from_secs(10);
+
+pub mod error;
+pub mod initialize;
+pub mod metrics;
+pub mod relay_loop;
+
+/// Block number traits shared by all chains that relay is able to serve.
+pub trait BlockNumberBase:
+	'static
+	+ From<u32>
+	+ UniqueSaturatedInto<u64>
+	+ Ord
+	+ Clone
+	+ Copy
+	+ Default
+	+ Send
+	+ Sync
+	+ std::fmt::Debug
+	+ std::fmt::Display
+	+ std::hash::Hash
+	+ std::ops::Add<Output = Self>
+	+ std::ops::Sub<Output = Self>
+	+ num_traits::CheckedSub
+	+ num_traits::Saturating
+	+ num_traits::Zero
+	+ num_traits::One
+{
+}
+
+impl<T> BlockNumberBase for T where
+	T: 'static
+		+ From<u32>
+		+ UniqueSaturatedInto<u64>
+		+ Ord
+		+ Clone
+		+ Copy
+		+ Default
+		+ Send
+		+ Sync
+		+ std::fmt::Debug
+		+ std::fmt::Display
+		+ std::hash::Hash
+		+ std::ops::Add<Output = Self>
+		+ std::ops::Sub<Output = Self>
+		+ num_traits::CheckedSub
+		+ num_traits::Saturating
+		+ num_traits::Zero
+		+ num_traits::One
+{
+}
+
+/// Macro that returns (client, Err(error)) tuple from function if result is Err(error).
+#[macro_export]
+macro_rules! bail_on_error {
+	($result: expr) => {
+		match $result {
+			(client, Ok(result)) => (client, result),
+			(client, Err(error)) => return (client, Err(error)),
+		}
+	};
+}
+
+/// Macro that returns (client, Err(error)) tuple from function if result is Err(error).
+#[macro_export]
+macro_rules! bail_on_arg_error {
+	($result: expr, $client: ident) => {
+		match $result {
+			Ok(result) => result,
+			Err(error) => return ($client, Err(error)),
+		}
+	};
+}
+
+/// Error type that can signal connection errors.
+pub trait MaybeConnectionError {
+	/// Returns true if error (maybe) represents connection error.
+	fn is_connection_error(&self) -> bool;
+}
+
+/// Final status of the tracked transaction.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum TrackedTransactionStatus<BlockId> {
+	/// Transaction has been lost.
+	Lost,
+	/// Transaction has been mined and finalized at given block.
+	Finalized(BlockId),
+}
+
+/// Transaction tracker.
+#[async_trait]
+pub trait TransactionTracker: Send {
+	/// Header id, used by the chain.
+	type HeaderId: Clone + Debug + Send;
+
+	/// Wait until transaction is either finalized or invalidated/lost.
+	async fn wait(self) -> TrackedTransactionStatus<Self::HeaderId>;
+}
+
+/// Future associated with `TransactionTracker`, monitoring the transaction status.
+pub type TrackedTransactionFuture<'a, T> =
+	BoxFuture<'a, TrackedTransactionStatus<<T as TransactionTracker>::HeaderId>>;
+
+/// Stringified error that may be either connection-related or not.
+#[derive(Error, Debug)]
+pub enum StringifiedMaybeConnectionError {
+	/// The error is connection-related error.
+	#[error("{0}")]
+	Connection(String),
+	/// The error is connection-unrelated error.
+	#[error("{0}")]
+	NonConnection(String),
+}
+
+impl StringifiedMaybeConnectionError {
+	/// Create new stringified connection error.
+	pub fn new(is_connection_error: bool, error: String) -> Self {
+		if is_connection_error {
+			StringifiedMaybeConnectionError::Connection(error)
+		} else {
+			StringifiedMaybeConnectionError::NonConnection(error)
+		}
+	}
+}
+
+impl MaybeConnectionError for StringifiedMaybeConnectionError {
+	fn is_connection_error(&self) -> bool {
+		match *self {
+			StringifiedMaybeConnectionError::Connection(_) => true,
+			StringifiedMaybeConnectionError::NonConnection(_) => false,
+		}
+	}
+}
+
+/// Exponential backoff for connection-unrelated errors retries.
+pub fn retry_backoff() -> ExponentialBackoff {
+	ExponentialBackoff {
+		// we do not want relayer to stop
+		max_elapsed_time: None,
+		max_interval: MAX_BACKOFF_INTERVAL,
+		..Default::default()
+	}
+}
+
+/// Compact format of IDs vector.
+pub fn format_ids<Id: std::fmt::Debug>(mut ids: impl ExactSizeIterator<Item = Id>) -> String {
+	const NTH_PROOF: &str = "we have checked len; qed";
+	match ids.len() {
+		0 => "<nothing>".into(),
+		1 => format!("{:?}", ids.next().expect(NTH_PROOF)),
+		2 => {
+			let id0 = ids.next().expect(NTH_PROOF);
+			let id1 = ids.next().expect(NTH_PROOF);
+			format!("[{id0:?}, {id1:?}]")
+		},
+		len => {
+			let id0 = ids.next().expect(NTH_PROOF);
+			let id_last = ids.last().expect(NTH_PROOF);
+			format!("{len}:[{id0:?} ... {id_last:?}]")
+		},
+	}
+}
+
+/// Stream that emits item every `timeout_ms` milliseconds.
+pub fn interval(timeout: Duration) -> impl futures::Stream<Item = ()> {
+	futures::stream::unfold((), move |_| async move {
+		async_std::task::sleep(timeout).await;
+		Some(((), ()))
+	})
+}
+
+/// Which client has caused error.
+#[derive(Debug, Eq, Clone, Copy, PartialEq)]
+pub enum FailedClient {
+	/// It is the source client who has caused error.
+	Source,
+	/// It is the target client who has caused error.
+	Target,
+	/// Both clients are failing, or we just encountered some other error that
+	/// should be treated like that.
+	Both,
+}
+
+/// Future process result.
+#[derive(Debug, Clone, Copy)]
+pub enum ProcessFutureResult {
+	/// Future has been processed successfully.
+	Success,
+	/// Future has failed with non-connection error.
+	Failed,
+	/// Future has failed with connection error.
+	ConnectionFailed,
+}
+
+impl ProcessFutureResult {
+	/// Returns true if result is Success.
+	pub fn is_ok(self) -> bool {
+		match self {
+			ProcessFutureResult::Success => true,
+			ProcessFutureResult::Failed | ProcessFutureResult::ConnectionFailed => false,
+		}
+	}
+
+	/// Returns `Ok(())` if future has succeeded.
+	/// Returns `Err(failed_client)` otherwise.
+	pub fn fail_if_error(self, failed_client: FailedClient) -> Result<(), FailedClient> {
+		if self.is_ok() {
+			Ok(())
+		} else {
+			Err(failed_client)
+		}
+	}
+
+	/// Returns Ok(true) if future has succeeded.
+	/// Returns Ok(false) if future has failed with non-connection error.
+	/// Returns Err if future is `ConnectionFailed`.
+	pub fn fail_if_connection_error(
+		self,
+		failed_client: FailedClient,
+	) -> Result<bool, FailedClient> {
+		match self {
+			ProcessFutureResult::Success => Ok(true),
+			ProcessFutureResult::Failed => Ok(false),
+			ProcessFutureResult::ConnectionFailed => Err(failed_client),
+		}
+	}
+}
+
+/// Process result of the future from a client.
+pub fn process_future_result<TResult, TError, TGoOfflineFuture>(
+	result: Result<TResult, TError>,
+	retry_backoff: &mut ExponentialBackoff,
+	on_success: impl FnOnce(TResult),
+	go_offline_future: &mut std::pin::Pin<&mut futures::future::Fuse<TGoOfflineFuture>>,
+	go_offline: impl FnOnce(Duration) -> TGoOfflineFuture,
+	error_pattern: impl FnOnce() -> String,
+) -> ProcessFutureResult
+where
+	TError: std::fmt::Debug + MaybeConnectionError,
+	TGoOfflineFuture: FutureExt,
+{
+	match result {
+		Ok(result) => {
+			on_success(result);
+			retry_backoff.reset();
+			ProcessFutureResult::Success
+		},
+		Err(error) if error.is_connection_error() => {
+			tracing::error!(
+				target: "bridge",
+				?error,
+				error_pattern=?error_pattern(),
+				"Going to restart"
+			);
+
+			retry_backoff.reset();
+			go_offline_future.set(go_offline(CONNECTION_ERROR_DELAY).fuse());
+			ProcessFutureResult::ConnectionFailed
+		},
+		Err(error) => {
+			let retry_delay = retry_backoff.next_backoff().unwrap_or(CONNECTION_ERROR_DELAY);
+			tracing::error!(
+				target: "bridge",
+				?error,
+				error_pattern=?error_pattern(),
+				retry_as_secs=%retry_delay.as_secs_f64(),
+				"Retrying"
+			);
+
+			go_offline_future.set(go_offline(retry_delay).fuse());
+			ProcessFutureResult::Failed
+		},
+	}
+}

pezbridges/relays/utils/src/metrics.rs

@@ -0,0 +1,192 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+pub use float_json_value::FloatJsonValueMetric;
+pub use global::GlobalMetrics;
+pub use prometheus_endpoint::{
+	prometheus::core::{Atomic, Collector},
+	register, Counter, CounterVec, Gauge, GaugeVec, Opts, PrometheusError, Registry, F64, I64, U64,
+};
+
+use async_std::sync::{Arc, RwLock};
+use async_trait::async_trait;
+use std::{fmt::Debug, time::Duration};
+
+mod float_json_value;
+mod global;
+
+/// Shared reference to `f64` value that is updated by the metric.
+pub type F64SharedRef = Arc<RwLock<Option<f64>>>;
+/// Int gauge metric type.
+pub type IntGauge = Gauge<U64>;
+
+/// Unparsed address that needs to be used to expose Prometheus metrics.
+#[derive(Debug, Clone)]
+pub struct MetricsAddress {
+	/// Serve HTTP requests at given host.
+	pub host: String,
+	/// Serve HTTP requests at given port.
+	pub port: u16,
+}
+
+/// Prometheus endpoint MetricsParams.
+#[derive(Debug, Clone)]
+pub struct MetricsParams {
+	/// Interface and TCP port to be used when exposing Prometheus metrics.
+	pub address: Option<MetricsAddress>,
+	/// Metrics registry. May be `Some(_)` if several components share the same endpoint.
+	pub registry: Registry,
+}
+
+/// Metric API.
+pub trait Metric: Clone + Send + Sync + 'static {
+	fn register(&self, registry: &Registry) -> Result<(), PrometheusError>;
+}
+
+/// Standalone metric API.
+///
+/// Metrics of this kind know how to update themselves, so we may just spawn and forget the
+/// asynchronous self-update task.
+#[async_trait]
+pub trait StandaloneMetric: Metric {
+	/// Update metric values.
+	async fn update(&self);
+
+	/// Metrics update interval.
+	fn update_interval(&self) -> Duration;
+
+	/// Register and spawn metric. Metric is only spawned if it is registered for the first time.
+	fn register_and_spawn(self, registry: &Registry) -> Result<(), PrometheusError> {
+		match self.register(registry) {
+			Ok(()) => {
+				self.spawn();
+				Ok(())
+			},
+			Err(PrometheusError::AlreadyReg) => Ok(()),
+			Err(e) => Err(e),
+		}
+	}
+
+	/// Spawn the self update task that will keep update metric value at given intervals.
+	fn spawn(self) {
+		async_std::task::spawn(async move {
+			let update_interval = self.update_interval();
+			loop {
+				self.update().await;
+				async_std::task::sleep(update_interval).await;
+			}
+		});
+	}
+}
+
+impl Default for MetricsAddress {
+	fn default() -> Self {
+		MetricsAddress { host: "127.0.0.1".into(), port: 9616 }
+	}
+}
+
+impl MetricsParams {
+	/// Creates metrics params from metrics address.
+	pub fn new(
+		address: Option<MetricsAddress>,
+		relay_version: String,
+		relay_commit: String,
+	) -> Result<Self, PrometheusError> {
+		const BUILD_INFO_METRIC: &str = "bizinikiwi_relay_build_info";
+
+		let registry = Registry::new();
+		register(
+			Gauge::<U64>::with_opts(
+				Opts::new(
+					BUILD_INFO_METRIC,
+					"A metric with a constant '1' value labeled by version",
+				)
+				.const_label("version", &relay_version)
+				.const_label("commit", &relay_commit),
+			)?,
+			&registry,
+		)?
+		.set(1);
+
+		tracing::info!(
+			target: "bridge",
+			metric=%BUILD_INFO_METRIC,
+			version=%relay_version,
+			commit=%relay_commit,
+			"Exposed metric"
+		);
+
+		Ok(MetricsParams { address, registry })
+	}
+
+	/// Creates metrics params so that metrics are not exposed.
+	pub fn disabled() -> Self {
+		MetricsParams { address: None, registry: Registry::new() }
+	}
+
+	/// Do not expose metrics.
+	#[must_use]
+	pub fn disable(mut self) -> Self {
+		self.address = None;
+		self
+	}
+}
+
+/// Returns metric name optionally prefixed with given prefix.
+pub fn metric_name(prefix: Option<&str>, name: &str) -> String {
+	if let Some(prefix) = prefix {
+		format!("{prefix}_{name}")
+	} else {
+		name.into()
+	}
+}
+
+/// Set value of gauge metric.
+///
+/// If value is `Ok(None)` or `Err(_)`, metric would have default value.
+pub fn set_gauge_value<T: Default + Debug, V: Atomic<T = T>, E: Debug>(
+	gauge: &Gauge<V>,
+	value: Result<Option<T>, E>,
+) {
+	gauge.set(match value {
+		Ok(Some(value)) => {
+			tracing::trace!(
+				target: "bridge-metrics",
+				metric=?gauge.desc().first().map(|d| &d.fq_name),
+				?value,
+				"Updated value"
+			);
+			value
+		},
+		Ok(None) => {
+			tracing::warn!(
+				target: "bridge-metrics",
+				metric=?gauge.desc().first().map(|d| &d.fq_name),
+				"Failed to update: value is empty"
+			);
+			Default::default()
+		},
+		Err(error) => {
+			tracing::warn!(
+				target: "bridge-metrics",
+				?error,
+				metric=?gauge.desc().first().map(|d| &d.fq_name),
+				"Failed to update"
+			);
+			Default::default()
+		},
+	})
+}

pezbridges/relays/utils/src/metrics/float_json_value.rs

@@ -0,0 +1,147 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{
+	error::{self, Error},
+	metrics::{
+		metric_name, register, F64SharedRef, Gauge, Metric, PrometheusError, Registry,
+		StandaloneMetric, F64,
+	},
+};
+
+use async_std::sync::{Arc, RwLock};
+use async_trait::async_trait;
+use std::time::Duration;
+
+/// Value update interval.
+const UPDATE_INTERVAL: Duration = Duration::from_secs(300);
+
+/// Metric that represents float value received from HTTP service as float gauge.
+///
+/// The float value returned by the service is assumed to be normal (`f64::is_normal`
+/// should return `true`) and strictly positive.
+#[derive(Debug, Clone)]
+pub struct FloatJsonValueMetric {
+	url: String,
+	json_path: String,
+	metric: Gauge<F64>,
+	shared_value_ref: F64SharedRef,
+}
+
+impl FloatJsonValueMetric {
+	/// Create new metric instance with given name and help.
+	pub fn new(
+		url: String,
+		json_path: String,
+		name: String,
+		help: String,
+	) -> Result<Self, PrometheusError> {
+		let shared_value_ref = Arc::new(RwLock::new(None));
+		Ok(FloatJsonValueMetric {
+			url,
+			json_path,
+			metric: Gauge::new(metric_name(None, &name), help)?,
+			shared_value_ref,
+		})
+	}
+
+	/// Get shared reference to metric value.
+	pub fn shared_value_ref(&self) -> F64SharedRef {
+		self.shared_value_ref.clone()
+	}
+
+	/// Request value from HTTP service.
+	async fn request_value(&self) -> anyhow::Result<String> {
+		use isahc::{AsyncReadResponseExt, HttpClient, Request};
+
+		let request = Request::get(&self.url).header("Accept", "application/json").body(())?;
+		let raw_response = HttpClient::new()?.send_async(request).await?.text().await?;
+		Ok(raw_response)
+	}
+
+	/// Read value from HTTP service.
+	async fn read_value(&self) -> error::Result<f64> {
+		let raw_response = self.request_value().await.map_err(Error::FetchTokenPrice)?;
+		parse_service_response(&self.json_path, &raw_response)
+	}
+}
+
+impl Metric for FloatJsonValueMetric {
+	fn register(&self, registry: &Registry) -> Result<(), PrometheusError> {
+		register(self.metric.clone(), registry).map(drop)
+	}
+}
+
+#[async_trait]
+impl StandaloneMetric for FloatJsonValueMetric {
+	fn update_interval(&self) -> Duration {
+		UPDATE_INTERVAL
+	}
+
+	async fn update(&self) {
+		let value = self.read_value().await;
+		let maybe_ok = value.as_ref().ok().copied();
+		crate::metrics::set_gauge_value(&self.metric, value.map(Some));
+		*self.shared_value_ref.write().await = maybe_ok;
+	}
+}
+
+/// Parse HTTP service response.
+fn parse_service_response(json_path: &str, response: &str) -> error::Result<f64> {
+	let json =
+		serde_json::from_str(response).map_err(|err| Error::ParseHttp(err, response.to_owned()))?;
+
+	let mut selector = jsonpath_lib::selector(&json);
+	let maybe_selected_value =
+		selector(json_path).map_err(|err| Error::SelectResponseValue(err, response.to_owned()))?;
+	let selected_value = maybe_selected_value
+		.first()
+		.and_then(|v| v.as_f64())
+		.ok_or_else(|| Error::MissingResponseValue(response.to_owned()))?;
+	if !selected_value.is_normal() || selected_value < 0.0 {
+		return Err(Error::ParseFloat(selected_value));
+	}
+
+	Ok(selected_value)
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn parse_service_response_works() {
+		assert_eq!(
+			parse_service_response("$.kusama.usd", r#"{"kusama":{"usd":433.05}}"#).map_err(drop),
+			Ok(433.05),
+		);
+	}
+
+	#[test]
+	fn parse_service_response_rejects_negative_numbers() {
+		assert!(parse_service_response("$.kusama.usd", r#"{"kusama":{"usd":-433.05}}"#).is_err());
+	}
+
+	#[test]
+	fn parse_service_response_rejects_zero_numbers() {
+		assert!(parse_service_response("$.kusama.usd", r#"{"kusama":{"usd":0.0}}"#).is_err());
+	}
+
+	#[test]
+	fn parse_service_response_rejects_nan() {
+		assert!(parse_service_response("$.kusama.usd", r#"{"kusama":{"usd":NaN}}"#).is_err());
+	}
+}

pezbridges/relays/utils/src/metrics/global.rs

@@ -0,0 +1,118 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Global system-wide Prometheus metrics exposed by relays.
+
+use crate::metrics::{
+	metric_name, register, Gauge, GaugeVec, Metric, Opts, PrometheusError, Registry,
+	StandaloneMetric, F64, U64,
+};
+
+use async_std::sync::{Arc, Mutex};
+use async_trait::async_trait;
+use std::time::Duration;
+use sysinfo::{RefreshKind, System};
+
+/// Global metrics update interval.
+const UPDATE_INTERVAL: Duration = Duration::from_secs(10);
+
+/// Global Prometheus metrics.
+#[derive(Debug, Clone)]
+pub struct GlobalMetrics {
+	system: Arc<Mutex<System>>,
+	system_average_load: GaugeVec<F64>,
+	process_cpu_usage_percentage: Gauge<F64>,
+	process_memory_usage_bytes: Gauge<U64>,
+}
+
+impl GlobalMetrics {
+	/// Create and register global metrics.
+	pub fn new() -> Result<Self, PrometheusError> {
+		Ok(GlobalMetrics {
+			system: Arc::new(Mutex::new(System::new_with_specifics(RefreshKind::everything()))),
+			system_average_load: GaugeVec::new(
+				Opts::new(metric_name(None, "system_average_load"), "System load average"),
+				&["over"],
+			)?,
+			process_cpu_usage_percentage: Gauge::new(
+				metric_name(None, "process_cpu_usage_percentage"),
+				"Process CPU usage",
+			)?,
+			process_memory_usage_bytes: Gauge::new(
+				metric_name(None, "process_memory_usage_bytes"),
+				"Process memory (resident set size) usage",
+			)?,
+		})
+	}
+}
+
+impl Metric for GlobalMetrics {
+	fn register(&self, registry: &Registry) -> Result<(), PrometheusError> {
+		register(self.system_average_load.clone(), registry)?;
+		register(self.process_cpu_usage_percentage.clone(), registry)?;
+		register(self.process_memory_usage_bytes.clone(), registry)?;
+		Ok(())
+	}
+}
+
+#[async_trait]
+impl StandaloneMetric for GlobalMetrics {
+	async fn update(&self) {
+		// update system-wide metrics
+		let mut system = self.system.lock().await;
+		let load = sysinfo::System::load_average();
+		self.system_average_load.with_label_values(&["1min"]).set(load.one);
+		self.system_average_load.with_label_values(&["5min"]).set(load.five);
+		self.system_average_load.with_label_values(&["15min"]).set(load.fifteen);
+
+		// update process-related metrics
+		let pid = sysinfo::get_current_pid().expect(
+			"only fails where pid is unavailable (os=unknown || arch=wasm32);\
+				relay is not supposed to run in such MetricsParamss;\
+				qed",
+		);
+		let is_process_refreshed = system.refresh_process(pid);
+		match (is_process_refreshed, system.process(pid)) {
+			(true, Some(process_info)) => {
+				let cpu_usage = process_info.cpu_usage() as f64;
+				let memory_usage = process_info.memory() * 1024;
+				tracing::trace!(
+					target: "bridge-metrics",
+					?cpu_usage,
+					?memory_usage,
+					"Refreshed process metrics"
+				);
+
+				self.process_cpu_usage_percentage.set(if cpu_usage.is_finite() {
+					cpu_usage
+				} else {
+					0f64
+				});
+				self.process_memory_usage_bytes.set(memory_usage);
+			},
+			_ => {
+				tracing::warn!(
+					target: "bridge-metrics",
+					"Failed to refresh process information. Metrics may show obsolete values"
+				);
+			},
+		}
+	}
+
+	fn update_interval(&self) -> Duration {
+		UPDATE_INTERVAL
+	}
+}

pezbridges/relays/utils/src/relay_loop.rs

@@ -0,0 +1,262 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{
+	error::Error,
+	metrics::{Metric, MetricsAddress, MetricsParams},
+	FailedClient, MaybeConnectionError,
+};
+
+use async_trait::async_trait;
+use prometheus_endpoint::{init_prometheus, Registry};
+use std::{fmt::Debug, future::Future, net::SocketAddr, time::Duration};
+
+/// Default pause between reconnect attempts.
+pub const RECONNECT_DELAY: Duration = Duration::from_secs(10);
+
+/// Basic blockchain client from relay perspective.
+#[async_trait]
+pub trait Client: 'static + Clone + Send + Sync {
+	/// Type of error these clients returns.
+	type Error: 'static + Debug + MaybeConnectionError + Send + Sync;
+
+	/// Try to reconnect to source node.
+	async fn reconnect(&mut self) -> Result<(), Self::Error>;
+
+	/// Try to reconnect to the source node in an infinite loop until it succeeds.
+	async fn reconnect_until_success(&mut self, delay: Duration) {
+		loop {
+			match self.reconnect().await {
+				Ok(()) => break,
+				Err(error) => {
+					tracing::warn!(
+						target: "bridge",
+						?error,
+						retry_as_secs=%delay.as_secs(),
+						"Failed to reconnect to client. Going to retry"
+					);
+
+					async_std::task::sleep(delay).await;
+				},
+			}
+		}
+	}
+}
+
+#[async_trait]
+impl Client for () {
+	type Error = crate::StringifiedMaybeConnectionError;
+
+	async fn reconnect(&mut self) -> Result<(), Self::Error> {
+		Ok(())
+	}
+}
+
+/// Returns generic loop that may be customized and started.
+pub fn relay_loop<SC, TC>(source_client: SC, target_client: TC) -> Loop<SC, TC, ()> {
+	Loop { reconnect_delay: RECONNECT_DELAY, source_client, target_client, loop_metric: None }
+}
+
+/// Returns generic relay loop metrics that may be customized and used in one or several relay
+/// loops.
+pub fn relay_metrics(params: MetricsParams) -> LoopMetrics<(), (), ()> {
+	LoopMetrics {
+		relay_loop: Loop {
+			reconnect_delay: RECONNECT_DELAY,
+			source_client: (),
+			target_client: (),
+			loop_metric: None,
+		},
+		address: params.address,
+		registry: params.registry,
+		loop_metric: None,
+	}
+}
+
+/// Generic relay loop.
+pub struct Loop<SC, TC, LM> {
+	reconnect_delay: Duration,
+	source_client: SC,
+	target_client: TC,
+	loop_metric: Option<LM>,
+}
+
+/// Relay loop metrics builder.
+pub struct LoopMetrics<SC, TC, LM> {
+	relay_loop: Loop<SC, TC, ()>,
+	address: Option<MetricsAddress>,
+	registry: Registry,
+	loop_metric: Option<LM>,
+}
+
+impl<SC, TC, LM> Loop<SC, TC, LM> {
+	/// Customize delay between reconnect attempts.
+	#[must_use]
+	pub fn reconnect_delay(mut self, reconnect_delay: Duration) -> Self {
+		self.reconnect_delay = reconnect_delay;
+		self
+	}
+
+	/// Start building loop metrics using given prefix.
+	pub fn with_metrics(self, params: MetricsParams) -> LoopMetrics<SC, TC, ()> {
+		LoopMetrics {
+			relay_loop: Loop {
+				reconnect_delay: self.reconnect_delay,
+				source_client: self.source_client,
+				target_client: self.target_client,
+				loop_metric: None,
+			},
+			address: params.address,
+			registry: params.registry,
+			loop_metric: None,
+		}
+	}
+
+	/// Run relay loop.
+	///
+	/// This function represents an outer loop, which in turn calls provided `run_loop` function to
+	/// do actual job. When `run_loop` returns, this outer loop reconnects to failed client (source,
+	/// target or both) and calls `run_loop` again.
+	pub async fn run<R, F>(mut self, loop_name: String, run_loop: R) -> Result<(), Error>
+	where
+		R: 'static + Send + Fn(SC, TC, Option<LM>) -> F,
+		F: 'static + Send + Future<Output = Result<(), FailedClient>>,
+		SC: 'static + Client,
+		TC: 'static + Client,
+		LM: 'static + Send + Clone,
+	{
+		let run_loop_task = async move {
+			crate::initialize::initialize_loop(loop_name);
+
+			loop {
+				let loop_metric = self.loop_metric.clone();
+				let future_result =
+					run_loop(self.source_client.clone(), self.target_client.clone(), loop_metric);
+				let result = future_result.await;
+
+				match result {
+					Ok(()) => break,
+					Err(failed_client) => {
+						tracing::debug!(target: "bridge", "Restarting relay loop");
+
+						reconnect_failed_client(
+							failed_client,
+							self.reconnect_delay,
+							&mut self.source_client,
+							&mut self.target_client,
+						)
+						.await
+					},
+				}
+			}
+			Ok(())
+		};
+
+		async_std::task::spawn(run_loop_task).await
+	}
+}
+
+impl<SC, TC, LM> LoopMetrics<SC, TC, LM> {
+	/// Add relay loop metrics.
+	///
+	/// Loop metrics will be passed to the loop callback.
+	pub fn loop_metric<NewLM: Metric>(
+		self,
+		metric: NewLM,
+	) -> Result<LoopMetrics<SC, TC, NewLM>, Error> {
+		metric.register(&self.registry)?;
+
+		Ok(LoopMetrics {
+			relay_loop: self.relay_loop,
+			address: self.address,
+			registry: self.registry,
+			loop_metric: Some(metric),
+		})
+	}
+
+	/// Convert into `MetricsParams` structure so that metrics registry may be extended later.
+	pub fn into_params(self) -> MetricsParams {
+		MetricsParams { address: self.address, registry: self.registry }
+	}
+
+	/// Expose metrics using address passed at creation.
+	///
+	/// If passed `address` is `None`, metrics are not exposed.
+	pub async fn expose(self) -> Result<Loop<SC, TC, LM>, Error> {
+		if let Some(address) = self.address {
+			let socket_addr = SocketAddr::new(
+				address
+					.host
+					.parse()
+					.map_err(|err| Error::ExposingMetricsInvalidHost(address.host.clone(), err))?,
+				address.port,
+			);
+
+			let registry = self.registry;
+			async_std::task::spawn(async move {
+				let runtime =
+					match tokio::runtime::Builder::new_current_thread().enable_all().build() {
+						Ok(runtime) => runtime,
+						Err(err) => {
+							tracing::trace!(
+								target: "bridge-metrics",
+								error=?err,
+								"Failed to create tokio runtime. Prometheus metrics are not available"
+							);
+							return;
+						},
+					};
+
+				runtime.block_on(async move {
+					tracing::trace!(
+						target: "bridge-metrics",
+						at=?socket_addr,
+						"Starting prometheus endpoint"
+					);
+					let result = init_prometheus(socket_addr, registry).await;
+					tracing::trace!(
+						target: "bridge-metrics",
+						?result,
+						"Prometheus endpoint has exited"
+					);
+				});
+			});
+		}
+
+		Ok(Loop {
+			reconnect_delay: self.relay_loop.reconnect_delay,
+			source_client: self.relay_loop.source_client,
+			target_client: self.relay_loop.target_client,
+			loop_metric: self.loop_metric,
+		})
+	}
+}
+
+/// Deal with the clients that have returned connection error.
+pub async fn reconnect_failed_client(
+	failed_client: FailedClient,
+	reconnect_delay: Duration,
+	source_client: &mut impl Client,
+	target_client: &mut impl Client,
+) {
+	if failed_client == FailedClient::Source || failed_client == FailedClient::Both {
+		source_client.reconnect_until_success(reconnect_delay).await;
+	}
+
+	if failed_client == FailedClient::Target || failed_client == FailedClient::Both {
+		target_client.reconnect_until_success(reconnect_delay).await;
+	}
+}