Relay balance metrics (#1291)

* relay balance metrics

* convert balance to "main" tokens in balance metrics

* add balances widgets to maintenance dashboard

bridges/relays/bin-substrate/src/cli/relay_headers_and_messages.rs

@@ -374,7 +374,7 @@ impl RelayHeadersAndMessages {
 			let right_to_left_metrics = left_to_right_metrics.clone().reverse();
 
 			// start conversion rate update loops for left/right chains
-			if let Some(left_messages_pallet_owner) = left_messages_pallet_owner {
+			if let Some(left_messages_pallet_owner) = left_messages_pallet_owner.clone() {
 				let left_client = left_client.clone();
 				let format_err = || {
 					anyhow::format_err!(
@@ -417,7 +417,7 @@ impl RelayHeadersAndMessages {
 					},
 				);
 			}
-			if let Some(right_messages_pallet_owner) = right_messages_pallet_owner {
+			if let Some(right_messages_pallet_owner) = right_messages_pallet_owner.clone() {
 				let right_client = right_client.clone();
 				let format_err = || {
 					anyhow::format_err!(
@@ -500,6 +500,24 @@ impl RelayHeadersAndMessages {
 				}
 			}
 
+			// add balance-related metrics
+			let metrics_params =
+				substrate_relay_helper::messages_metrics::add_relay_balances_metrics(
+					left_client.clone(),
+					metrics_params,
+					Some(left_sign.public().into()),
+					left_messages_pallet_owner.map(|kp| kp.public().into()),
+				)
+				.await?;
+			let metrics_params =
+				substrate_relay_helper::messages_metrics::add_relay_balances_metrics(
+					right_client.clone(),
+					metrics_params,
+					Some(right_sign.public().into()),
+					right_messages_pallet_owner.map(|kp| kp.public().into()),
+				)
+				.await?;
+
 			// start on-demand header relays
 			let left_to_right_transaction_params = TransactionParams {
 				mortality: right_transactions_mortality,

bridges/relays/client-substrate/Cargo.toml

@@ -33,9 +33,10 @@ frame-system = { git = "https://github.com/paritytech/substrate", branch = "mast
 pallet-balances = { git = "https://github.com/paritytech/substrate", branch = "master" }
 pallet-transaction-payment = { git = "https://github.com/paritytech/substrate", branch = "master" }
 pallet-transaction-payment-rpc-runtime-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sc-chain-spec = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-rpc-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
-sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-transaction-pool-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-finality-grandpa = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-rpc = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }

bridges/relays/client-substrate/src/chain.rs

@@ -117,7 +117,7 @@ pub type WeightToFeeOf<C> = <C as Chain>::WeightToFee;
 /// Transaction status of the chain.
 pub type TransactionStatusOf<C> = TransactionStatus<HashOf<C>, HashOf<C>>;
 
-/// Substrate-based chain with `frame_system::Config::AccountData` set to
+/// Substrate-based chain with `AccountData` generic argument of `frame_system::AccountInfo` set to
 /// the `pallet_balances::AccountData<Balance>`.
 pub trait ChainWithBalances: Chain {
 	/// Return runtime storage key for getting `frame_system::AccountInfo` of given account.

bridges/relays/client-substrate/src/client.rs

@@ -541,6 +541,15 @@ impl<C: Chain> Client<C> {
 		.await
 	}
 
+	/// Return `tokenDecimals` property from the set of chain properties.
+	pub async fn token_decimals(&self) -> Result<Option<u64>> {
+		self.jsonrpsee_execute(move |client| async move {
+			let system_properties = Substrate::<C>::system_properties(&*client).await?;
+			Ok(system_properties.get("tokenDecimals").and_then(|v| v.as_u64()))
+		})
+		.await
+	}
+
 	/// Return new justifications stream.
 	pub async fn subscribe_justifications(&self) -> Result<Subscription<Bytes>> {
 		let subscription = self

bridges/relays/client-substrate/src/metrics/float_storage_value.rs

@@ -14,48 +14,84 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
 
-use crate::{chain::Chain, client::Client};
+use crate::{chain::Chain, client::Client, Error as SubstrateError};
 
 use async_std::sync::{Arc, RwLock};
 use async_trait::async_trait;
 use codec::Decode;
+use num_traits::One;
 use relay_utils::metrics::{
 	metric_name, register, F64SharedRef, Gauge, Metric, PrometheusError, Registry,
 	StandaloneMetric, F64,
 };
-use sp_core::storage::StorageKey;
-use sp_runtime::{traits::UniqueSaturatedInto, FixedPointNumber};
-use std::time::Duration;
+use sp_core::storage::{StorageData, StorageKey};
+use sp_runtime::{traits::UniqueSaturatedInto, FixedPointNumber, FixedU128};
+use std::{marker::PhantomData, time::Duration};
 
 /// Storage value update interval (in blocks).
 const UPDATE_INTERVAL_IN_BLOCKS: u32 = 5;
 
-/// Metric that represents fixed-point runtime storage value as float gauge.
-#[derive(Clone, Debug)]
-pub struct FloatStorageValueMetric<C: Chain, T: Clone> {
-	client: Client<C>,
-	storage_key: StorageKey,
-	maybe_default_value: Option<T>,
-	metric: Gauge<F64>,
-	shared_value_ref: F64SharedRef,
+/// Fied-point storage value and the way it is decoded from the raw storage value.
+pub trait FloatStorageValue: 'static + Clone + Send + Sync {
+	/// Type of the value.
+	type Value: FixedPointNumber;
+	/// Try to decode value from the raw storage value.
+	fn decode(
+		&self,
+		maybe_raw_value: Option<StorageData>,
+	) -> Result<Option<Self::Value>, SubstrateError>;
 }
 
-impl<C: Chain, T: Decode + FixedPointNumber> FloatStorageValueMetric<C, T> {
+/// Implementation of `FloatStorageValue` that expects encoded `FixedU128` value and returns `1` if
+/// value is missing from the storage.
+#[derive(Clone, Debug, Default)]
+pub struct FixedU128OrOne;
+
+impl FloatStorageValue for FixedU128OrOne {
+	type Value = FixedU128;
+
+	fn decode(
+		&self,
+		maybe_raw_value: Option<StorageData>,
+	) -> Result<Option<Self::Value>, SubstrateError> {
+		maybe_raw_value
+			.map(|raw_value| {
+				FixedU128::decode(&mut &raw_value.0[..])
+					.map_err(SubstrateError::ResponseParseFailed)
+					.map(Some)
+			})
+			.unwrap_or_else(|| Ok(Some(FixedU128::one())))
+	}
+}
+
+/// Metric that represents fixed-point runtime storage value as float gauge.
+#[derive(Clone, Debug)]
+pub struct FloatStorageValueMetric<C: Chain, V: FloatStorageValue> {
+	value_converter: V,
+	client: Client<C>,
+	storage_key: StorageKey,
+	metric: Gauge<F64>,
+	shared_value_ref: F64SharedRef,
+	_phantom: PhantomData<V>,
+}
+
+impl<C: Chain, V: FloatStorageValue> FloatStorageValueMetric<C, V> {
 	/// Create new metric.
 	pub fn new(
+		value_converter: V,
 		client: Client<C>,
 		storage_key: StorageKey,
-		maybe_default_value: Option<T>,
 		name: String,
 		help: String,
 	) -> Result<Self, PrometheusError> {
 		let shared_value_ref = Arc::new(RwLock::new(None));
 		Ok(FloatStorageValueMetric {
+			value_converter,
 			client,
 			storage_key,
-			maybe_default_value,
 			metric: Gauge::new(metric_name(None, &name), help)?,
 			shared_value_ref,
+			_phantom: Default::default(),
 		})
 	}
 
@@ -65,20 +101,14 @@ impl<C: Chain, T: Decode + FixedPointNumber> FloatStorageValueMetric<C, T> {
 	}
 }
 
-impl<C: Chain, T> Metric for FloatStorageValueMetric<C, T>
-where
-	T: 'static + Decode + Send + Sync + FixedPointNumber,
-{
+impl<C: Chain, V: FloatStorageValue> Metric for FloatStorageValueMetric<C, V> {
 	fn register(&self, registry: &Registry) -> Result<(), PrometheusError> {
 		register(self.metric.clone(), registry).map(drop)
 	}
 }
 
 #[async_trait]
-impl<C: Chain, T> StandaloneMetric for FloatStorageValueMetric<C, T>
-where
-	T: 'static + Decode + Send + Sync + FixedPointNumber,
-{
+impl<C: Chain, V: FloatStorageValue> StandaloneMetric for FloatStorageValueMetric<C, V> {
 	fn update_interval(&self) -> Duration {
 		C::AVERAGE_BLOCK_INTERVAL * UPDATE_INTERVAL_IN_BLOCKS
 	}
@@ -86,16 +116,18 @@ where
 	async fn update(&self) {
 		let value = self
 			.client
-			.storage_value::<T>(self.storage_key.clone(), None)
+			.raw_storage_value(self.storage_key.clone(), None)
 			.await
-			.map(|maybe_storage_value| {
-				maybe_storage_value.or(self.maybe_default_value).map(|storage_value| {
-					storage_value.into_inner().unique_saturated_into() as f64 /
-						T::DIV.unique_saturated_into() as f64
+			.and_then(|maybe_storage_value| {
+				self.value_converter.decode(maybe_storage_value).map(|maybe_fixed_point_value| {
+					maybe_fixed_point_value.map(|fixed_point_value| {
+						fixed_point_value.into_inner().unique_saturated_into() as f64 /
+							V::Value::DIV.unique_saturated_into() as f64
+					})
 				})
 			})
-			.map_err(drop);
-		relay_utils::metrics::set_gauge_value(&self.metric, value);
+			.map_err(|e| e.to_string());
+		relay_utils::metrics::set_gauge_value(&self.metric, value.clone());
 		*self.shared_value_ref.write().await = value.ok().and_then(|x| x);
 	}
 }

bridges/relays/client-substrate/src/metrics/mod.rs

@@ -16,7 +16,7 @@
 
 //! Contains several Substrate-specific metrics that may be exposed by relay.
 
-pub use float_storage_value::FloatStorageValueMetric;
+pub use float_storage_value::{FixedU128OrOne, FloatStorageValue, FloatStorageValueMetric};
 pub use storage_proof_overhead::StorageProofOverheadMetric;
 
 mod float_storage_value;

bridges/relays/client-substrate/src/rpc.rs

@@ -31,6 +31,8 @@ jsonrpsee_proc_macros::rpc_client_api! {
 	pub(crate) Substrate<C: Chain> {
 		#[rpc(method = "system_health", positional_params)]
 		fn system_health() -> Health;
+		#[rpc(method = "system_properties", positional_params)]
+		fn system_properties() -> sc_chain_spec::Properties;
 		#[rpc(method = "chain_getHeader", positional_params)]
 		fn chain_get_header(block_hash: Option<C::Hash>) -> C::Header;
 		#[rpc(method = "chain_getFinalizedHead", positional_params)]

bridges/relays/lib-substrate-relay/Cargo.toml

@@ -36,6 +36,8 @@ bp-messages = { path = "../../primitives/messages" }
 # Substrate Dependencies
 
 frame-support = { git = "https://github.com/paritytech/substrate", branch = "master" }
+frame-system = { git = "https://github.com/paritytech/substrate", branch = "master" }
+pallet-balances = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-finality-grandpa = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }

bridges/relays/lib-substrate-relay/src/messages_metrics.rs

@@ -18,16 +18,21 @@
 
 use crate::messages_lane::SubstrateMessageLane;
 
-use num_traits::One;
+use codec::Decode;
+use frame_system::AccountInfo;
+use pallet_balances::AccountData;
 use relay_substrate_client::{
-	metrics::{FloatStorageValueMetric, StorageProofOverheadMetric},
-	Chain, Client,
+	metrics::{
+		FixedU128OrOne, FloatStorageValue, FloatStorageValueMetric, StorageProofOverheadMetric,
+	},
+	AccountIdOf, BalanceOf, Chain, ChainWithBalances, Client, Error as SubstrateError, IndexOf,
 };
 use relay_utils::metrics::{
 	FloatJsonValueMetric, GlobalMetrics, MetricsParams, PrometheusError, StandaloneMetric,
 };
-use sp_runtime::FixedU128;
-use std::fmt::Debug;
+use sp_core::storage::StorageData;
+use sp_runtime::{FixedPointNumber, FixedU128};
+use std::{convert::TryFrom, fmt::Debug, marker::PhantomData};
 
 /// Shared references to the standalone metrics of the message lane relay loop.
 #[derive(Debug, Clone)]
@@ -44,12 +49,10 @@ pub struct StandaloneMessagesMetrics<SC: Chain, TC: Chain> {
 	pub target_to_base_conversion_rate: Option<FloatJsonValueMetric>,
 	/// Source tokens to target tokens conversion rate metric. This rate is stored by the target
 	/// chain.
-	pub source_to_target_conversion_rate:
-		Option<FloatStorageValueMetric<TC, sp_runtime::FixedU128>>,
+	pub source_to_target_conversion_rate: Option<FloatStorageValueMetric<TC, FixedU128OrOne>>,
 	/// Target tokens to source tokens conversion rate metric. This rate is stored by the source
 	/// chain.
-	pub target_to_source_conversion_rate:
-		Option<FloatStorageValueMetric<SC, sp_runtime::FixedU128>>,
+	pub target_to_source_conversion_rate: Option<FloatStorageValueMetric<SC, FixedU128OrOne>>,
 }
 
 impl<SC: Chain, TC: Chain> StandaloneMessagesMetrics<SC, TC> {
@@ -104,7 +107,7 @@ impl<SC: Chain, TC: Chain> StandaloneMessagesMetrics<SC, TC> {
 	}
 }
 
-/// Create standalone metrics for the message lane relay loop.
+/// Create symmetric standalone metrics for the message lane relay loop.
 ///
 /// All metrics returned by this function are exposed by loops that are serving given lane (`P`)
 /// and by loops that are serving reverse lane (`P` with swapped `TargetChain` and `SourceChain`).
@@ -139,10 +142,10 @@ pub fn standalone_metrics<P: SubstrateMessageLane>(
 		source_to_target_conversion_rate: P::SOURCE_TO_TARGET_CONVERSION_RATE_PARAMETER_NAME
 			.map(bp_runtime::storage_parameter_key)
 			.map(|key| {
-				FloatStorageValueMetric::<_, sp_runtime::FixedU128>::new(
+				FloatStorageValueMetric::new(
+					FixedU128OrOne::default(),
 					target_client,
 					key,
-					Some(FixedU128::one()),
 					format!(
 						"{}_{}_to_{}_conversion_rate",
 						P::TargetChain::NAME,
@@ -162,10 +165,10 @@ pub fn standalone_metrics<P: SubstrateMessageLane>(
 		target_to_source_conversion_rate: P::TARGET_TO_SOURCE_CONVERSION_RATE_PARAMETER_NAME
 			.map(bp_runtime::storage_parameter_key)
 			.map(|key| {
-				FloatStorageValueMetric::<_, sp_runtime::FixedU128>::new(
+				FloatStorageValueMetric::new(
+					FixedU128OrOne::default(),
 					source_client,
 					key,
-					Some(FixedU128::one()),
 					format!(
 						"{}_{}_to_{}_conversion_rate",
 						P::SourceChain::NAME,
@@ -185,6 +188,90 @@ pub fn standalone_metrics<P: SubstrateMessageLane>(
 	})
 }
 
+/// Add relay accounts balance metrics.
+pub async fn add_relay_balances_metrics<C: ChainWithBalances>(
+	client: Client<C>,
+	metrics: MetricsParams,
+	relay_account_id: Option<AccountIdOf<C>>,
+	messages_pallet_owner_account_id: Option<AccountIdOf<C>>,
+) -> anyhow::Result<MetricsParams>
+where
+	BalanceOf<C>: Into<u128> + std::fmt::Debug,
+{
+	if relay_account_id.is_none() && messages_pallet_owner_account_id.is_none() {
+		return Ok(metrics)
+	}
+
+	let token_decimals = client.token_decimals().await?.ok_or_else(|| {
+		SubstrateError::Custom(format!("Missing token decimals from {} system properties", C::NAME))
+	})?;
+	let token_decimals = u32::try_from(token_decimals).map_err(|e| {
+		anyhow::format_err!(
+			"Token decimals value ({}) of {} doesn't fit into u32: {:?}",
+			token_decimals,
+			C::NAME,
+			e,
+		)
+	})?;
+	if let Some(relay_account_id) = relay_account_id {
+		let relay_account_balance_metric = FloatStorageValueMetric::new(
+			FreeAccountBalance::<C> { token_decimals, _phantom: Default::default() },
+			client.clone(),
+			C::account_info_storage_key(&relay_account_id),
+			format!("at_{}_relay_balance", C::NAME),
+			format!("Balance of the relay account at the {}", C::NAME),
+		)?;
+		relay_account_balance_metric.register_and_spawn(&metrics.registry)?;
+	}
+	if let Some(messages_pallet_owner_account_id) = messages_pallet_owner_account_id {
+		let pallet_owner_account_balance_metric = FloatStorageValueMetric::new(
+			FreeAccountBalance::<C> { token_decimals, _phantom: Default::default() },
+			client.clone(),
+			C::account_info_storage_key(&messages_pallet_owner_account_id),
+			format!("at_{}_messages_pallet_owner_balance", C::NAME),
+			format!("Balance of the messages pallet owner at the {}", C::NAME),
+		)?;
+		pallet_owner_account_balance_metric.register_and_spawn(&metrics.registry)?;
+	}
+	Ok(metrics)
+}
+
+/// Adapter for `FloatStorageValueMetric` to decode account free balance.
+#[derive(Clone, Debug)]
+struct FreeAccountBalance<C> {
+	token_decimals: u32,
+	_phantom: PhantomData<C>,
+}
+
+impl<C> FloatStorageValue for FreeAccountBalance<C>
+where
+	C: Chain,
+	BalanceOf<C>: Into<u128>,
+{
+	type Value = FixedU128;
+
+	fn decode(
+		&self,
+		maybe_raw_value: Option<StorageData>,
+	) -> Result<Option<Self::Value>, SubstrateError> {
+		maybe_raw_value
+			.map(|raw_value| {
+				AccountInfo::<IndexOf<C>, AccountData<BalanceOf<C>>>::decode(&mut &raw_value.0[..])
+					.map_err(SubstrateError::ResponseParseFailed)
+					.map(|account_data| {
+						convert_to_token_balance(account_data.data.free.into(), self.token_decimals)
+					})
+			})
+			.transpose()
+	}
+}
+
+/// Convert from raw `u128` balance (nominated in smallest chain token units) to the float regular
+/// tokens value.
+fn convert_to_token_balance(balance: u128, token_decimals: u32) -> FixedU128 {
+	FixedU128::from_inner(balance.saturating_mul(FixedU128::DIV / 10u128.pow(token_decimals)))
+}
+
 #[cfg(test)]
 mod tests {
 	use super::*;
@@ -196,4 +283,12 @@ mod tests {
 			Some(12.32 / 183.15),
 		);
 	}
+
+	#[test]
+	fn token_decimals_used_properly() {
+		let plancks = 425_000_000_000;
+		let token_decimals = 10;
+		let dots = convert_to_token_balance(plancks, token_decimals);
+		assert_eq!(dots, FixedU128::saturating_from_rational(425, 10));
+	}
 }