Using-same-fork metric for finality and complex relay (#1327)

* using_same_fork metric in finality relay

* support `using_different_forks` in messages relay

* added dashboards and alerts

* lockfile

bridges/relays/lib-substrate-relay/src/finality_target.rs

@@ -30,8 +30,8 @@ use bp_header_chain::{justification::GrandpaJustification, storage_keys::is_halt
 use codec::Encode;
 use finality_relay::TargetClient;
 use relay_substrate_client::{
-	AccountIdOf, AccountKeyPairOf, BlockNumberOf, Chain, ChainWithGrandpa, Client, Error, HashOf,
-	HeaderOf, SignParam, SyncHeader, TransactionEra, TransactionSignScheme, UnsignedTransaction,
+	AccountIdOf, AccountKeyPairOf, Chain, ChainWithGrandpa, Client, Error, HeaderIdOf, HeaderOf,
+	SignParam, SyncHeader, TransactionEra, TransactionSignScheme, UnsignedTransaction,
 };
 use relay_utils::relay_loop::Client as RelayClient;
 use sp_core::{Bytes, Pair};
@@ -90,23 +90,20 @@ where
 	AccountIdOf<P::TargetChain>: From<<AccountKeyPairOf<P::TransactionSignScheme> as Pair>::Public>,
 	P::TransactionSignScheme: TransactionSignScheme<Chain = P::TargetChain>,
 {
-	async fn best_finalized_source_block_number(
-		&self,
-	) -> Result<BlockNumberOf<P::SourceChain>, Error> {
+	async fn best_finalized_source_block_id(&self) -> Result<HeaderIdOf<P::SourceChain>, Error> {
 		// we can't continue to relay finality if target node is out of sync, because
 		// it may have already received (some of) headers that we're going to relay
 		self.client.ensure_synced().await?;
 		// we can't relay finality if GRANDPA pallet at target chain is halted
 		self.ensure_pallet_active().await?;
 
-		Ok(crate::messages_source::read_client_state::<
-			P::TargetChain,
-			HashOf<P::SourceChain>,
-			BlockNumberOf<P::SourceChain>,
-		>(&self.client, P::SourceChain::BEST_FINALIZED_HEADER_ID_METHOD)
+		Ok(crate::messages_source::read_client_state::<P::TargetChain, P::SourceChain>(
+			&self.client,
+			None,
+			P::SourceChain::BEST_FINALIZED_HEADER_ID_METHOD,
+		)
 		.await?
-		.best_finalized_peer_at_best_self
-		.0)
+		.best_finalized_peer_at_best_self)
 	}
 
 	async fn submit_finality_proof(

bridges/relays/lib-substrate-relay/src/messages_lane.rs

@@ -234,13 +234,15 @@ where
 			},
 		},
 		SubstrateMessagesSource::<P>::new(
-			source_client,
+			source_client.clone(),
+			target_client.clone(),
 			params.lane_id,
 			params.source_transaction_params,
 			params.target_to_source_headers_relay,
 		),
 		SubstrateMessagesTarget::<P>::new(
 			target_client,
+			source_client,
 			params.lane_id,
 			relayer_id_at_source,
 			params.target_transaction_params,

bridges/relays/lib-substrate-relay/src/messages_source.rs

@@ -46,7 +46,7 @@ use messages_relay::{
 };
 use num_traits::{Bounded, Zero};
 use relay_substrate_client::{
-	AccountIdOf, AccountKeyPairOf, BalanceOf, Chain, ChainWithMessages, Client,
+	AccountIdOf, AccountKeyPairOf, BalanceOf, BlockNumberOf, Chain, ChainWithMessages, Client,
 	Error as SubstrateError, HashOf, HeaderIdOf, IndexOf, SignParam, TransactionEra,
 	TransactionSignScheme, UnsignedTransaction,
 };
@@ -62,7 +62,8 @@ pub type SubstrateMessagesProof<C> = (Weight, FromBridgedChainMessagesProof<Hash
 
 /// Substrate client as Substrate messages source.
 pub struct SubstrateMessagesSource<P: SubstrateMessageLane> {
-	client: Client<P::SourceChain>,
+	source_client: Client<P::SourceChain>,
+	target_client: Client<P::TargetChain>,
 	lane_id: LaneId,
 	transaction_params: TransactionParams<AccountKeyPairOf<P::SourceTransactionSignScheme>>,
 	target_to_source_headers_relay: Option<OnDemandHeadersRelay<P::TargetChain>>,
@@ -71,13 +72,15 @@ pub struct SubstrateMessagesSource<P: SubstrateMessageLane> {
 impl<P: SubstrateMessageLane> SubstrateMessagesSource<P> {
 	/// Create new Substrate headers source.
 	pub fn new(
-		client: Client<P::SourceChain>,
+		source_client: Client<P::SourceChain>,
+		target_client: Client<P::TargetChain>,
 		lane_id: LaneId,
 		transaction_params: TransactionParams<AccountKeyPairOf<P::SourceTransactionSignScheme>>,
 		target_to_source_headers_relay: Option<OnDemandHeadersRelay<P::TargetChain>>,
 	) -> Self {
 		SubstrateMessagesSource {
-			client,
+			source_client,
+			target_client,
 			lane_id,
 			transaction_params,
 			target_to_source_headers_relay,
@@ -89,7 +92,7 @@ impl<P: SubstrateMessageLane> SubstrateMessagesSource<P> {
 		&self,
 		id: SourceHeaderIdOf<MessageLaneAdapter<P>>,
 	) -> Result<Option<OutboundLaneData>, SubstrateError> {
-		self.client
+		self.source_client
 			.storage_value(
 				outbound_lane_data_key(
 					P::TargetChain::WITH_CHAIN_MESSAGES_PALLET_NAME,
@@ -102,14 +105,15 @@ impl<P: SubstrateMessageLane> SubstrateMessagesSource<P> {
 
 	/// Ensure that the messages pallet at source chain is active.
 	async fn ensure_pallet_active(&self) -> Result<(), SubstrateError> {
-		ensure_messages_pallet_active::<P::SourceChain, P::TargetChain>(&self.client).await
+		ensure_messages_pallet_active::<P::SourceChain, P::TargetChain>(&self.source_client).await
 	}
 }
 
 impl<P: SubstrateMessageLane> Clone for SubstrateMessagesSource<P> {
 	fn clone(&self) -> Self {
 		Self {
-			client: self.client.clone(),
+			source_client: self.source_client.clone(),
+			target_client: self.target_client.clone(),
 			lane_id: self.lane_id,
 			transaction_params: self.transaction_params.clone(),
 			target_to_source_headers_relay: self.target_to_source_headers_relay.clone(),
@@ -122,7 +126,8 @@ impl<P: SubstrateMessageLane> RelayClient for SubstrateMessagesSource<P> {
 	type Error = SubstrateError;
 
 	async fn reconnect(&mut self) -> Result<(), SubstrateError> {
-		self.client.reconnect().await
+		self.source_client.reconnect().await?;
+		self.target_client.reconnect().await
 	}
 }
 
@@ -136,15 +141,15 @@ where
 	async fn state(&self) -> Result<SourceClientState<MessageLaneAdapter<P>>, SubstrateError> {
 		// we can't continue to deliver confirmations if source node is out of sync, because
 		// it may have already received confirmations that we're going to deliver
-		self.client.ensure_synced().await?;
+		self.source_client.ensure_synced().await?;
 		// we can't relay confirmations if messages pallet at source chain is halted
 		self.ensure_pallet_active().await?;
 
-		read_client_state::<
-			_,
-			<MessageLaneAdapter<P> as MessageLane>::TargetHeaderHash,
-			<MessageLaneAdapter<P> as MessageLane>::TargetHeaderNumber,
-		>(&self.client, P::TargetChain::BEST_FINALIZED_HEADER_ID_METHOD)
+		read_client_state(
+			&self.source_client,
+			Some(&self.target_client),
+			P::TargetChain::BEST_FINALIZED_HEADER_ID_METHOD,
+		)
 		.await
 	}
 
@@ -183,7 +188,7 @@ where
 		SubstrateError,
 	> {
 		let encoded_response = self
-			.client
+			.source_client
 			.state_call(
 				P::TargetChain::TO_CHAIN_MESSAGE_DETAILS_METHOD.into(),
 				Bytes((self.lane_id, nonces.start(), nonces.end()).encode()),
@@ -230,7 +235,12 @@ where
 			));
 		}
 
-		let proof = self.client.prove_storage(storage_keys, id.1).await?.iter_nodes().collect();
+		let proof = self
+			.source_client
+			.prove_storage(storage_keys, id.1)
+			.await?
+			.iter_nodes()
+			.collect();
 		let proof = FromBridgedChainMessagesProof {
 			bridged_header_hash: id.1,
 			storage_proof: proof,
@@ -246,10 +256,11 @@ where
 		_generated_at_block: TargetHeaderIdOf<MessageLaneAdapter<P>>,
 		proof: <MessageLaneAdapter<P> as MessageLane>::MessagesReceivingProof,
 	) -> Result<(), SubstrateError> {
-		let genesis_hash = *self.client.genesis_hash();
+		let genesis_hash = *self.source_client.genesis_hash();
 		let transaction_params = self.transaction_params.clone();
-		let (spec_version, transaction_version) = self.client.simple_runtime_version().await?;
-		self.client
+		let (spec_version, transaction_version) =
+			self.source_client.simple_runtime_version().await?;
+		self.source_client
 			.submit_signed_extrinsic(
 				self.transaction_params.signer.public().into(),
 				move |best_block_id, transaction_nonce| {
@@ -278,7 +289,7 @@ where
 	async fn estimate_confirmation_transaction(
 		&self,
 	) -> <MessageLaneAdapter<P> as MessageLane>::SourceChainBalance {
-		let runtime_version = match self.client.runtime_version().await {
+		let runtime_version = match self.source_client.runtime_version().await {
 			Ok(v) => v,
 			Err(_) => return BalanceOf::<P::SourceChain>::max_value(),
 		};
@@ -286,14 +297,14 @@ where
 			let dummy_tx = make_messages_delivery_proof_transaction::<P>(
 				runtime_version.spec_version,
 				runtime_version.transaction_version,
-				self.client.genesis_hash(),
+				self.source_client.genesis_hash(),
 				&self.transaction_params,
 				HeaderId(Default::default(), Default::default()),
 				Zero::zero(),
 				prepare_dummy_messages_delivery_proof::<P::SourceChain, P::TargetChain>(),
 				false,
 			)?;
-			self.client
+			self.source_client
 				.estimate_extrinsic_fee(dummy_tx)
 				.await
 				.map(|fee| fee.inclusion_fee())
@@ -385,19 +396,19 @@ fn prepare_dummy_messages_delivery_proof<SC: Chain, TC: Chain>(
 /// This function assumes that the chain that is followed by the `self_client` has
 /// bridge GRANDPA pallet deployed and it provides `best_finalized_header_id_method_name`
 /// runtime API to read the best finalized Bridged chain header.
-pub async fn read_client_state<SelfChain, BridgedHeaderHash, BridgedHeaderNumber>(
+///
+/// If `peer_client` is `None`, the value of `actual_best_finalized_peer_at_best_self` will
+/// always match the `best_finalized_peer_at_best_self`.
+pub async fn read_client_state<SelfChain, PeerChain>(
 	self_client: &Client<SelfChain>,
+	peer_client: Option<&Client<PeerChain>>,
 	best_finalized_header_id_method_name: &str,
-) -> Result<
-	ClientState<HeaderIdOf<SelfChain>, HeaderId<BridgedHeaderHash, BridgedHeaderNumber>>,
-	SubstrateError,
->
+) -> Result<ClientState<HeaderIdOf<SelfChain>, HeaderIdOf<PeerChain>>, SubstrateError>
 where
 	SelfChain: Chain,
 	SelfChain::Header: DeserializeOwned,
 	SelfChain::Index: DeserializeOwned,
-	BridgedHeaderHash: Decode,
-	BridgedHeaderNumber: Decode,
+	PeerChain: Chain,
 {
 	// let's read our state first: we need best finalized header hash on **this** chain
 	let self_best_finalized_header_hash = self_client.best_finalized_header_hash().await?;
@@ -419,16 +430,27 @@ where
 			Some(self_best_hash),
 		)
 		.await?;
-	let decoded_best_finalized_peer_on_self: (BridgedHeaderNumber, BridgedHeaderHash) =
+	let decoded_best_finalized_peer_on_self: (BlockNumberOf<PeerChain>, HashOf<PeerChain>) =
 		Decode::decode(&mut &encoded_best_finalized_peer_on_self.0[..])
 			.map_err(SubstrateError::ResponseParseFailed)?;
 	let peer_on_self_best_finalized_id =
 		HeaderId(decoded_best_finalized_peer_on_self.0, decoded_best_finalized_peer_on_self.1);
 
+	// read actual header, matching the `peer_on_self_best_finalized_id` from the peer chain
+	let actual_peer_on_self_best_finalized_id = match peer_client {
+		Some(peer_client) => {
+			let actual_peer_on_self_best_finalized =
+				peer_client.header_by_number(peer_on_self_best_finalized_id.0).await?;
+			HeaderId(peer_on_self_best_finalized_id.0, actual_peer_on_self_best_finalized.hash())
+		},
+		None => peer_on_self_best_finalized_id.clone(),
+	};
+
 	Ok(ClientState {
 		best_self: self_best_id,
 		best_finalized_self: self_best_finalized_id,
 		best_finalized_peer_at_best_self: peer_on_self_best_finalized_id,
+		actual_best_finalized_peer_at_best_self: actual_peer_on_self_best_finalized_id,
 	})
 }
 

bridges/relays/lib-substrate-relay/src/messages_target.rs

@@ -57,7 +57,8 @@ pub type SubstrateMessagesDeliveryProof<C> =
 
 /// Substrate client as Substrate messages target.
 pub struct SubstrateMessagesTarget<P: SubstrateMessageLane> {
-	client: Client<P::TargetChain>,
+	target_client: Client<P::TargetChain>,
+	source_client: Client<P::SourceChain>,
 	lane_id: LaneId,
 	relayer_id_at_source: AccountIdOf<P::SourceChain>,
 	transaction_params: TransactionParams<AccountKeyPairOf<P::TargetTransactionSignScheme>>,
@@ -68,7 +69,8 @@ pub struct SubstrateMessagesTarget<P: SubstrateMessageLane> {
 impl<P: SubstrateMessageLane> SubstrateMessagesTarget<P> {
 	/// Create new Substrate headers target.
 	pub fn new(
-		client: Client<P::TargetChain>,
+		target_client: Client<P::TargetChain>,
+		source_client: Client<P::SourceChain>,
 		lane_id: LaneId,
 		relayer_id_at_source: AccountIdOf<P::SourceChain>,
 		transaction_params: TransactionParams<AccountKeyPairOf<P::TargetTransactionSignScheme>>,
@@ -76,7 +78,8 @@ impl<P: SubstrateMessageLane> SubstrateMessagesTarget<P> {
 		source_to_target_headers_relay: Option<OnDemandHeadersRelay<P::SourceChain>>,
 	) -> Self {
 		SubstrateMessagesTarget {
-			client,
+			target_client,
+			source_client,
 			lane_id,
 			relayer_id_at_source,
 			transaction_params,
@@ -90,7 +93,7 @@ impl<P: SubstrateMessageLane> SubstrateMessagesTarget<P> {
 		&self,
 		id: TargetHeaderIdOf<MessageLaneAdapter<P>>,
 	) -> Result<Option<InboundLaneData<AccountIdOf<P::SourceChain>>>, SubstrateError> {
-		self.client
+		self.target_client
 			.storage_value(
 				inbound_lane_data_key(
 					P::SourceChain::WITH_CHAIN_MESSAGES_PALLET_NAME,
@@ -103,14 +106,15 @@ impl<P: SubstrateMessageLane> SubstrateMessagesTarget<P> {
 
 	/// Ensure that the messages pallet at target chain is active.
 	async fn ensure_pallet_active(&self) -> Result<(), SubstrateError> {
-		ensure_messages_pallet_active::<P::TargetChain, P::SourceChain>(&self.client).await
+		ensure_messages_pallet_active::<P::TargetChain, P::SourceChain>(&self.target_client).await
 	}
 }
 
 impl<P: SubstrateMessageLane> Clone for SubstrateMessagesTarget<P> {
 	fn clone(&self) -> Self {
 		Self {
-			client: self.client.clone(),
+			target_client: self.target_client.clone(),
+			source_client: self.source_client.clone(),
 			lane_id: self.lane_id,
 			relayer_id_at_source: self.relayer_id_at_source.clone(),
 			transaction_params: self.transaction_params.clone(),
@@ -125,7 +129,8 @@ impl<P: SubstrateMessageLane> RelayClient for SubstrateMessagesTarget<P> {
 	type Error = SubstrateError;
 
 	async fn reconnect(&mut self) -> Result<(), SubstrateError> {
-		self.client.reconnect().await
+		self.target_client.reconnect().await?;
+		self.source_client.reconnect().await
 	}
 }
 
@@ -140,15 +145,15 @@ where
 	async fn state(&self) -> Result<TargetClientState<MessageLaneAdapter<P>>, SubstrateError> {
 		// we can't continue to deliver messages if target node is out of sync, because
 		// it may have already received (some of) messages that we're going to deliver
-		self.client.ensure_synced().await?;
+		self.target_client.ensure_synced().await?;
 		// we can't relay messages if messages pallet at target chain is halted
 		self.ensure_pallet_active().await?;
 
-		read_client_state::<
-			_,
-			<MessageLaneAdapter<P> as MessageLane>::SourceHeaderHash,
-			<MessageLaneAdapter<P> as MessageLane>::SourceHeaderNumber,
-		>(&self.client, P::SourceChain::BEST_FINALIZED_HEADER_ID_METHOD)
+		read_client_state(
+			&self.target_client,
+			Some(&self.source_client),
+			P::SourceChain::BEST_FINALIZED_HEADER_ID_METHOD,
+		)
 		.await
 	}
 
@@ -184,7 +189,7 @@ where
 	) -> Result<(TargetHeaderIdOf<MessageLaneAdapter<P>>, UnrewardedRelayersState), SubstrateError>
 	{
 		let encoded_response = self
-			.client
+			.target_client
 			.state_call(
 				P::SourceChain::FROM_CHAIN_UNREWARDED_RELAYERS_STATE.into(),
 				Bytes(self.lane_id.encode()),
@@ -213,7 +218,7 @@ where
 			&self.lane_id,
 		);
 		let proof = self
-			.client
+			.target_client
 			.prove_storage(vec![inbound_data_key], id.1)
 			.await?
 			.iter_nodes()
@@ -232,12 +237,13 @@ where
 		nonces: RangeInclusive<MessageNonce>,
 		proof: <MessageLaneAdapter<P> as MessageLane>::MessagesProof,
 	) -> Result<RangeInclusive<MessageNonce>, SubstrateError> {
-		let genesis_hash = *self.client.genesis_hash();
+		let genesis_hash = *self.target_client.genesis_hash();
 		let transaction_params = self.transaction_params.clone();
 		let relayer_id_at_source = self.relayer_id_at_source.clone();
 		let nonces_clone = nonces.clone();
-		let (spec_version, transaction_version) = self.client.simple_runtime_version().await?;
-		self.client
+		let (spec_version, transaction_version) =
+			self.target_client.simple_runtime_version().await?;
+		self.target_client
 			.submit_signed_extrinsic(
 				self.transaction_params.signer.public().into(),
 				move |best_block_id, transaction_nonce| {
@@ -281,12 +287,13 @@ where
 				))
 			})?;
 
-		let (spec_version, transaction_version) = self.client.simple_runtime_version().await?;
+		let (spec_version, transaction_version) =
+			self.target_client.simple_runtime_version().await?;
 		// Prepare 'dummy' delivery transaction - we only care about its length and dispatch weight.
 		let delivery_tx = make_messages_delivery_transaction::<P>(
 			spec_version,
 			transaction_version,
-			self.client.genesis_hash(),
+			self.target_client.genesis_hash(),
 			&self.transaction_params,
 			HeaderId(Default::default(), Default::default()),
 			Zero::zero(),
@@ -299,7 +306,7 @@ where
 			),
 			false,
 		)?;
-		let delivery_tx_fee = self.client.estimate_extrinsic_fee(delivery_tx).await?;
+		let delivery_tx_fee = self.target_client.estimate_extrinsic_fee(delivery_tx).await?;
 		let inclusion_fee_in_target_tokens = delivery_tx_fee.inclusion_fee();
 
 		// The pre-dispatch cost of delivery transaction includes additional fee to cover dispatch
@@ -321,12 +328,13 @@ where
 		let expected_refund_in_target_tokens = if total_prepaid_nonces != 0 {
 			const WEIGHT_DIFFERENCE: Weight = 100;
 
-			let (spec_version, transaction_version) = self.client.simple_runtime_version().await?;
+			let (spec_version, transaction_version) =
+				self.target_client.simple_runtime_version().await?;
 			let larger_dispatch_weight = total_dispatch_weight.saturating_add(WEIGHT_DIFFERENCE);
 			let dummy_tx = make_messages_delivery_transaction::<P>(
 				spec_version,
 				transaction_version,
-				self.client.genesis_hash(),
+				self.target_client.genesis_hash(),
 				&self.transaction_params,
 				HeaderId(Default::default(), Default::default()),
 				Zero::zero(),
@@ -339,7 +347,8 @@ where
 				),
 				false,
 			)?;
-			let larger_delivery_tx_fee = self.client.estimate_extrinsic_fee(dummy_tx).await?;
+			let larger_delivery_tx_fee =
+				self.target_client.estimate_extrinsic_fee(dummy_tx).await?;
 
 			compute_prepaid_messages_refund::<P::TargetChain>(
 				total_prepaid_nonces,

bridges/relays/lib-substrate-relay/src/on_demand_headers.rs

@@ -382,16 +382,20 @@ where
 		From<<AccountKeyPairOf<P::TransactionSignScheme> as sp_core::Pair>::Public>,
 	P::TransactionSignScheme: TransactionSignScheme<Chain = P::TargetChain>,
 {
-	finality_target.best_finalized_source_block_number().await.map_err(|error| {
-		log::error!(
-			target: "bridge",
-			"Failed to read best finalized source header from target in {} relay: {:?}",
-			relay_task_name,
-			error,
-		);
+	finality_target
+		.best_finalized_source_block_id()
+		.await
+		.map_err(|error| {
+			log::error!(
+				target: "bridge",
+				"Failed to read best finalized source header from target in {} relay: {:?}",
+				relay_task_name,
+				error,
+			);
 
-		error
-	})
+			error
+		})
+		.map(|id| id.0)
 }
 
 /// Read first mandatory header in given inclusive range.