fix: Complete snowbridge pezpallet rebrand and critical bug fixes

- snowbridge-pezpallet-* → pezsnowbridge-pezpallet-* (201 refs)
- pallet/ directories → pezpallet/ (4 locations)
- Fixed pezpallet.rs self-include recursion bug
- Fixed sc-chain-spec hardcoded crate name in derive macro
- Reverted .pezpallet_by_name() to .pallet_by_name() (subxt API)
- Added BizinikiwiConfig type alias for zombienet tests
- Deleted obsolete session state files

Verified: pezsnowbridge-pezpallet-*, pezpallet-staking,
pezpallet-staking-async, pezframe-benchmarking-cli all pass cargo check

pezbridges/relays/messages/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+name = "pez-messages-relay"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+repository.workspace = true
+publish = false
+description = "Pezkuwi SDK component: messages relay"
+documentation = "https://docs.rs/pez-messages-relay"
+homepage = { workspace = true }
+
+[lints]
+workspace = true
+
+[dependencies]
+async-std = { features = ["attributes"], workspace = true }
+async-trait = { workspace = true }
+futures = { workspace = true }
+hex = { workspace = true, default-features = true }
+num-traits = { workspace = true, default-features = true }
+parking_lot = { workspace = true, default-features = true }
+tracing = { workspace = true }
+
+# Bridge Dependencies
+bp-messages = { workspace = true, default-features = true }
+pez-finality-relay = { workspace = true }
+relay-utils = { workspace = true }
+
+pezsp-arithmetic = { workspace = true, default-features = true }
+
+[dev-dependencies]
+pezsp-core = { workspace = true }
+
+[features]
+runtime-benchmarks = [
+  "bp-messages/runtime-benchmarks",
+  "pez-finality-relay/runtime-benchmarks",
+  "relay-utils/runtime-benchmarks",
+]

pezbridges/relays/messages/src/lib.rs

@@ -0,0 +1,41 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Relaying [`pezpallet-bridge-messages`](../pezpallet_bridge_messages/index.html) application specific
+//! data. Message lane allows sending arbitrary messages between bridged chains. This
+//! module provides entrypoint that starts reading messages from given message lane
+//! of source chain and submits proof-of-message-at-source-chain transactions to the
+//! target chain. Additionally, proofs-of-messages-delivery are sent back from the
+//! target chain to the source chain.
+
+// required for futures::select!
+#![recursion_limit = "1024"]
+#![warn(missing_docs)]
+
+mod metrics;
+
+pub mod message_lane;
+pub mod message_lane_loop;
+
+mod message_race_delivery;
+mod message_race_limits;
+mod message_race_loop;
+mod message_race_receiving;
+mod message_race_strategy;
+
+pub use message_race_delivery::relay_messages_range;
+pub use message_race_receiving::relay_messages_delivery_confirmation;
+pub use metrics::Labeled;

pezbridges/relays/messages/src/message_lane.rs

@@ -0,0 +1,75 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! One-way message lane types. Within single one-way lane we have three 'races' where we try to:
+//!
+//! 1) relay new messages from source to target node;
+//! 2) relay proof-of-delivery from target to source node.
+
+use crate::metrics::Labeled;
+use num_traits::{SaturatingAdd, Zero};
+use relay_utils::{BlockNumberBase, HeaderId};
+use pezsp_arithmetic::traits::AtLeast32BitUnsigned;
+use std::{fmt::Debug, ops::Sub};
+
+/// One-way message lane.
+pub trait MessageLane: 'static + Clone + Send + Sync {
+	/// Name of the messages source.
+	const SOURCE_NAME: &'static str;
+	/// Name of the messages target.
+	const TARGET_NAME: &'static str;
+
+	/// Lane identifier type.
+	type LaneId: Clone + Send + Sync + Labeled;
+
+	/// Messages proof.
+	type MessagesProof: Clone + Debug + Send + Sync;
+	/// Messages receiving proof.
+	type MessagesReceivingProof: Clone + Debug + Send + Sync;
+
+	/// The type of the source chain token balance, that is used to:
+	///
+	/// 1) pay transaction fees;
+	/// 2) pay message delivery and dispatch fee;
+	/// 3) pay relayer rewards.
+	type SourceChainBalance: AtLeast32BitUnsigned
+		+ Clone
+		+ Copy
+		+ Debug
+		+ PartialOrd
+		+ Sub<Output = Self::SourceChainBalance>
+		+ SaturatingAdd
+		+ Zero
+		+ Send
+		+ Sync;
+	/// Number of the source header.
+	type SourceHeaderNumber: BlockNumberBase;
+	/// Hash of the source header.
+	type SourceHeaderHash: Clone + Debug + Default + PartialEq + Send + Sync;
+
+	/// Number of the target header.
+	type TargetHeaderNumber: BlockNumberBase;
+	/// Hash of the target header.
+	type TargetHeaderHash: Clone + Debug + Default + PartialEq + Send + Sync;
+}
+
+/// Source header id within given one-way message lane.
+pub type SourceHeaderIdOf<P> =
+	HeaderId<<P as MessageLane>::SourceHeaderHash, <P as MessageLane>::SourceHeaderNumber>;
+
+/// Target header id within given one-way message lane.
+pub type TargetHeaderIdOf<P> =
+	HeaderId<<P as MessageLane>::TargetHeaderHash, <P as MessageLane>::TargetHeaderNumber>;

pezbridges/relays/messages/src/message_race_limits.rs

@@ -0,0 +1,169 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! enforcement strategy
+
+use num_traits::Zero;
+use std::ops::RangeInclusive;
+
+use bp_messages::{MessageNonce, Weight};
+
+use crate::{
+	message_lane::MessageLane,
+	message_lane_loop::{MessageDetails, MessageDetailsMap},
+	message_race_loop::NoncesRange,
+	message_race_strategy::SourceRangesQueue,
+};
+
+/// Reference data for participating in relay
+pub struct RelayReference<P: MessageLane> {
+	/// Messages size summary
+	pub selected_size: u32,
+
+	/// Index by all ready nonces
+	pub index: usize,
+	/// Current nonce
+	pub nonce: MessageNonce,
+	/// Current nonce details
+	pub details: MessageDetails<P::SourceChainBalance>,
+}
+
+/// Relay reference data
+pub struct RelayMessagesBatchReference<P: MessageLane> {
+	/// Maximal number of relayed messages in single delivery transaction.
+	pub max_messages_in_this_batch: MessageNonce,
+	/// Maximal cumulative dispatch weight of relayed messages in single delivery transaction.
+	pub max_messages_weight_in_single_batch: Weight,
+	/// Maximal cumulative size of relayed messages in single delivery transaction.
+	pub max_messages_size_in_single_batch: u32,
+	/// Best available nonce at the **best** target block. We do not want to deliver nonces
+	/// less than this nonce, even though the block may be retracted.
+	pub best_target_nonce: MessageNonce,
+	/// Source queue.
+	pub nonces_queue: SourceRangesQueue<
+		P::SourceHeaderHash,
+		P::SourceHeaderNumber,
+		MessageDetailsMap<P::SourceChainBalance>,
+	>,
+	/// Range of indices within the `nonces_queue` that are available for selection.
+	pub nonces_queue_range: RangeInclusive<usize>,
+}
+
+/// Limits of the message race transactions.
+#[derive(Clone)]
+pub struct MessageRaceLimits;
+
+impl MessageRaceLimits {
+	pub async fn decide<P: MessageLane>(
+		reference: RelayMessagesBatchReference<P>,
+	) -> Option<RangeInclusive<MessageNonce>> {
+		let mut hard_selected_count = 0;
+
+		let mut selected_weight = Weight::zero();
+		let mut selected_count: MessageNonce = 0;
+
+		let hard_selected_begin_nonce = std::cmp::max(
+			reference.best_target_nonce + 1,
+			reference.nonces_queue[*reference.nonces_queue_range.start()].1.begin(),
+		);
+
+		// relay reference
+		let mut relay_reference = RelayReference::<P> {
+			selected_size: 0,
+
+			index: 0,
+			nonce: 0,
+			details: MessageDetails {
+				dispatch_weight: Weight::zero(),
+				size: 0,
+				reward: P::SourceChainBalance::zero(),
+			},
+		};
+
+		let all_ready_nonces = reference
+			.nonces_queue
+			.range(reference.nonces_queue_range.clone())
+			.flat_map(|(_, ready_nonces)| ready_nonces.iter())
+			.filter(|(nonce, _)| **nonce >= hard_selected_begin_nonce)
+			.enumerate();
+		for (index, (nonce, details)) in all_ready_nonces {
+			relay_reference.index = index;
+			relay_reference.nonce = *nonce;
+			relay_reference.details = *details;
+
+			// Since we (hopefully) have some reserves in `max_messages_weight_in_single_batch`
+			// and `max_messages_size_in_single_batch`, we may still try to submit transaction
+			// with single message if message overflows these limits. The worst case would be if
+			// transaction will be rejected by the target runtime, but at least we have tried.
+
+			// limit messages in the batch by weight
+			let new_selected_weight = match selected_weight.checked_add(&details.dispatch_weight) {
+				Some(new_selected_weight)
+					if new_selected_weight
+						.all_lte(reference.max_messages_weight_in_single_batch) =>
+					new_selected_weight,
+				new_selected_weight if selected_count == 0 => {
+					tracing::warn!(
+						target: "bridge",
+						dispatch_weight=?new_selected_weight,
+						configured_weight=%reference.max_messages_weight_in_single_batch,
+						"Going to submit message delivery transaction with declared dispatch \
+						weight that overflows maximal configured weight"
+					);
+					new_selected_weight.unwrap_or(Weight::MAX)
+				},
+				_ => break,
+			};
+
+			// limit messages in the batch by size
+			let new_selected_size = match relay_reference.selected_size.checked_add(details.size) {
+				Some(new_selected_size)
+					if new_selected_size <= reference.max_messages_size_in_single_batch =>
+					new_selected_size,
+				new_selected_size if selected_count == 0 => {
+					tracing::warn!(
+						target: "bridge",
+						message_size=new_selected_size,
+						configured_size=%reference.max_messages_size_in_single_batch,
+						"Going to submit message delivery transaction with message \
+						size that overflows maximal configured size"
+					);
+					new_selected_size.unwrap_or(u32::MAX)
+				},
+				_ => break,
+			};
+
+			// limit number of messages in the batch
+			let new_selected_count = selected_count + 1;
+			if new_selected_count > reference.max_messages_in_this_batch {
+				break;
+			}
+			relay_reference.selected_size = new_selected_size;
+
+			hard_selected_count = index + 1;
+			selected_weight = new_selected_weight;
+			selected_count = new_selected_count;
+		}
+
+		if hard_selected_count != 0 {
+			let selected_max_nonce =
+				hard_selected_begin_nonce + hard_selected_count as MessageNonce - 1;
+			Some(hard_selected_begin_nonce..=selected_max_nonce)
+		} else {
+			None
+		}
+	}
+}

pezbridges/relays/messages/src/message_race_loop.rs

@@ -0,0 +1,816 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+//! Loop that is serving single race within message lane. This could be
+//! message delivery race, receiving confirmations race or processing
+//! confirmations race.
+//!
+//! The idea of the race is simple - we have `nonce`-s on source and target
+//! nodes. We're trying to prove that the source node has this nonce (and
+//! associated data - like messages, lane state, etc) to the target node by
+//! generating and submitting proof.
+
+use crate::message_lane_loop::{BatchTransaction, ClientState, NoncesSubmitArtifacts};
+
+use async_trait::async_trait;
+use bp_messages::MessageNonce;
+use futures::{
+	future::{FutureExt, TryFutureExt},
+	stream::{FusedStream, StreamExt},
+};
+use relay_utils::{
+	process_future_result, retry_backoff, FailedClient, MaybeConnectionError,
+	TrackedTransactionStatus, TransactionTracker,
+};
+use std::{
+	fmt::Debug,
+	ops::RangeInclusive,
+	time::{Duration, Instant},
+};
+
+/// One of races within lane.
+pub trait MessageRace {
+	/// Header id of the race source.
+	type SourceHeaderId: Debug + Clone + PartialEq + Send + Sync;
+	/// Header id of the race source.
+	type TargetHeaderId: Debug + Clone + PartialEq + Send + Sync;
+
+	/// Message nonce used in the race.
+	type MessageNonce: Debug + Clone;
+	/// Proof that is generated and delivered in this race.
+	type Proof: Debug + Clone + Send + Sync;
+
+	/// Name of the race source.
+	fn source_name() -> String;
+	/// Name of the race target.
+	fn target_name() -> String;
+}
+
+/// State of race source client.
+type SourceClientState<P> =
+	ClientState<<P as MessageRace>::SourceHeaderId, <P as MessageRace>::TargetHeaderId>;
+
+/// State of race target client.
+type TargetClientState<P> =
+	ClientState<<P as MessageRace>::TargetHeaderId, <P as MessageRace>::SourceHeaderId>;
+
+/// Inclusive nonces range.
+pub trait NoncesRange: Debug + Sized {
+	/// Get begin of the range.
+	fn begin(&self) -> MessageNonce;
+	/// Get end of the range.
+	fn end(&self) -> MessageNonce;
+	/// Returns new range with current range nonces that are greater than the passed `nonce`.
+	/// If there are no such nonces, `None` is returned.
+	fn greater_than(self, nonce: MessageNonce) -> Option<Self>;
+}
+
+/// Nonces on the race source client.
+#[derive(Debug, Clone)]
+pub struct SourceClientNonces<NoncesRange> {
+	/// New nonces range known to the client. `New` here means all nonces generated after
+	/// `prev_latest_nonce` passed to the `SourceClient::nonces` method.
+	pub new_nonces: NoncesRange,
+	/// The latest nonce that is confirmed to the bridged client. This nonce only makes
+	/// sense in some races. In other races it is `None`.
+	pub confirmed_nonce: Option<MessageNonce>,
+}
+
+/// Nonces on the race target client.
+#[derive(Debug, Clone)]
+pub struct TargetClientNonces<TargetNoncesData> {
+	/// The latest nonce that is known to the target client.
+	pub latest_nonce: MessageNonce,
+	/// Additional data from target node that may be used by the race.
+	pub nonces_data: TargetNoncesData,
+}
+
+/// One of message lane clients, which is source client for the race.
+#[async_trait]
+pub trait SourceClient<P: MessageRace> {
+	/// Type of error these clients returns.
+	type Error: std::fmt::Debug + MaybeConnectionError;
+	/// Type of nonces range returned by the source client.
+	type NoncesRange: NoncesRange;
+	/// Additional proof parameters required to generate proof.
+	type ProofParameters;
+
+	/// Return nonces that are known to the source client.
+	async fn nonces(
+		&self,
+		at_block: P::SourceHeaderId,
+		prev_latest_nonce: MessageNonce,
+	) -> Result<(P::SourceHeaderId, SourceClientNonces<Self::NoncesRange>), Self::Error>;
+	/// Generate proof for delivering to the target client.
+	async fn generate_proof(
+		&self,
+		at_block: P::SourceHeaderId,
+		nonces: RangeInclusive<MessageNonce>,
+		proof_parameters: Self::ProofParameters,
+	) -> Result<(P::SourceHeaderId, RangeInclusive<MessageNonce>, P::Proof), Self::Error>;
+}
+
+/// One of message lane clients, which is target client for the race.
+#[async_trait]
+pub trait TargetClient<P: MessageRace> {
+	/// Type of error these clients returns.
+	type Error: std::fmt::Debug + MaybeConnectionError;
+	/// Type of the additional data from the target client, used by the race.
+	type TargetNoncesData: std::fmt::Debug;
+	/// Type of batch transaction that submits finality and proof to the target node.
+	type BatchTransaction: BatchTransaction<P::SourceHeaderId> + Clone;
+	/// Transaction tracker to track submitted transactions.
+	type TransactionTracker: TransactionTracker<HeaderId = P::TargetHeaderId>;
+
+	/// Ask headers relay to relay finalized headers up to (and including) given header
+	/// from race source to race target.
+	///
+	/// The client may return `Some(_)`, which means that nothing has happened yet and
+	/// the caller must generate and append proof to the batch transaction
+	/// to actually send it (along with required header) to the node.
+	///
+	/// If function has returned `None`, it means that the caller now must wait for the
+	/// appearance of the required header `id` at the target client.
+	async fn require_source_header(
+		&self,
+		id: P::SourceHeaderId,
+	) -> Result<Option<Self::BatchTransaction>, Self::Error>;
+
+	/// Return nonces that are known to the target client.
+	async fn nonces(
+		&self,
+		at_block: P::TargetHeaderId,
+		update_metrics: bool,
+	) -> Result<(P::TargetHeaderId, TargetClientNonces<Self::TargetNoncesData>), Self::Error>;
+	/// Submit proof to the target client.
+	async fn submit_proof(
+		&self,
+		maybe_batch_tx: Option<Self::BatchTransaction>,
+		generated_at_block: P::SourceHeaderId,
+		nonces: RangeInclusive<MessageNonce>,
+		proof: P::Proof,
+	) -> Result<NoncesSubmitArtifacts<Self::TransactionTracker>, Self::Error>;
+}
+
+/// Race strategy.
+#[async_trait]
+pub trait RaceStrategy<SourceHeaderId, TargetHeaderId, Proof>: Debug {
+	/// Type of nonces range expected from the source client.
+	type SourceNoncesRange: NoncesRange;
+	/// Additional proof parameters required to generate proof.
+	type ProofParameters;
+	/// Additional data expected from the target client.
+	type TargetNoncesData;
+
+	/// Return id of source header that is required to be on target to continue synchronization.
+	async fn required_source_header_at_target<RS: RaceState<SourceHeaderId, TargetHeaderId>>(
+		&self,
+		race_state: RS,
+	) -> Option<SourceHeaderId>;
+	/// Return the best nonce at source node.
+	///
+	/// `Some` is returned only if we are sure that the value is greater or equal
+	/// than the result of `best_at_target`.
+	fn best_at_source(&self) -> Option<MessageNonce>;
+	/// Return the best nonce at target node.
+	///
+	/// May return `None` if value is yet unknown.
+	fn best_at_target(&self) -> Option<MessageNonce>;
+
+	/// Called when nonces are updated at source node of the race.
+	fn source_nonces_updated(
+		&mut self,
+		at_block: SourceHeaderId,
+		nonces: SourceClientNonces<Self::SourceNoncesRange>,
+	);
+	/// Called when we want to wait until next `best_target_nonces_updated` before selecting
+	/// any nonces for delivery.
+	fn reset_best_target_nonces(&mut self);
+	/// Called when best nonces are updated at target node of the race.
+	fn best_target_nonces_updated<RS: RaceState<SourceHeaderId, TargetHeaderId>>(
+		&mut self,
+		nonces: TargetClientNonces<Self::TargetNoncesData>,
+		race_state: &mut RS,
+	);
+	/// Called when finalized nonces are updated at target node of the race.
+	fn finalized_target_nonces_updated<RS: RaceState<SourceHeaderId, TargetHeaderId>>(
+		&mut self,
+		nonces: TargetClientNonces<Self::TargetNoncesData>,
+		race_state: &mut RS,
+	);
+	/// Should return `Some(nonces)` if we need to deliver proof of `nonces` (and associated
+	/// data) from source to target node.
+	/// Additionally, parameters required to generate proof are returned.
+	async fn select_nonces_to_deliver<RS: RaceState<SourceHeaderId, TargetHeaderId>>(
+		&self,
+		race_state: RS,
+	) -> Option<(RangeInclusive<MessageNonce>, Self::ProofParameters)>;
+}
+
+/// State of the race.
+pub trait RaceState<SourceHeaderId, TargetHeaderId>: Clone + Send + Sync {
+	/// Set best finalized source header id at the best block on the target
+	/// client (at the `best_finalized_source_header_id_at_best_target`).
+	fn set_best_finalized_source_header_id_at_best_target(&mut self, id: SourceHeaderId);
+
+	/// Best finalized source header id at the best block on the target
+	/// client (at the `best_finalized_source_header_id_at_best_target`).
+	fn best_finalized_source_header_id_at_best_target(&self) -> Option<SourceHeaderId>;
+
+	/// Returns `true` if we have selected nonces to submit to the target node.
+	fn nonces_to_submit(&self) -> Option<RangeInclusive<MessageNonce>>;
+	/// Reset our nonces selection.
+	fn reset_nonces_to_submit(&mut self);
+
+	/// Returns `true` if we have submitted some nonces to the target node and are
+	/// waiting for them to appear there.
+	fn nonces_submitted(&self) -> Option<RangeInclusive<MessageNonce>>;
+	/// Reset our nonces submission.
+	fn reset_nonces_submitted(&mut self);
+}
+
+/// State of the race and prepared batch transaction (if available).
+#[derive(Debug, Clone)]
+pub(crate) struct RaceStateImpl<SourceHeaderId, TargetHeaderId, Proof, BatchTx> {
+	/// Best finalized source header id at the source client.
+	pub best_finalized_source_header_id_at_source: Option<SourceHeaderId>,
+	/// Best finalized source header id at the best block on the target
+	/// client (at the `best_finalized_source_header_id_at_best_target`).
+	pub best_finalized_source_header_id_at_best_target: Option<SourceHeaderId>,
+	/// The best header id at the target client.
+	pub best_target_header_id: Option<TargetHeaderId>,
+	/// Best finalized header id at the target client.
+	pub best_finalized_target_header_id: Option<TargetHeaderId>,
+	/// Range of nonces that we have selected to submit.
+	pub nonces_to_submit: Option<(SourceHeaderId, RangeInclusive<MessageNonce>, Proof)>,
+	/// Batch transaction ready to include and deliver selected `nonces_to_submit` from the
+	/// `state`.
+	pub nonces_to_submit_batch: Option<BatchTx>,
+	/// Range of nonces that is currently submitted.
+	pub nonces_submitted: Option<RangeInclusive<MessageNonce>>,
+}
+
+impl<SourceHeaderId, TargetHeaderId, Proof, BatchTx> Default
+	for RaceStateImpl<SourceHeaderId, TargetHeaderId, Proof, BatchTx>
+{
+	fn default() -> Self {
+		RaceStateImpl {
+			best_finalized_source_header_id_at_source: None,
+			best_finalized_source_header_id_at_best_target: None,
+			best_target_header_id: None,
+			best_finalized_target_header_id: None,
+			nonces_to_submit: None,
+			nonces_to_submit_batch: None,
+			nonces_submitted: None,
+		}
+	}
+}
+
+impl<SourceHeaderId, TargetHeaderId, Proof, BatchTx> RaceState<SourceHeaderId, TargetHeaderId>
+	for RaceStateImpl<SourceHeaderId, TargetHeaderId, Proof, BatchTx>
+where
+	SourceHeaderId: Clone + Send + Sync,
+	TargetHeaderId: Clone + Send + Sync,
+	Proof: Clone + Send + Sync,
+	BatchTx: Clone + Send + Sync,
+{
+	fn set_best_finalized_source_header_id_at_best_target(&mut self, id: SourceHeaderId) {
+		self.best_finalized_source_header_id_at_best_target = Some(id);
+	}
+
+	fn best_finalized_source_header_id_at_best_target(&self) -> Option<SourceHeaderId> {
+		self.best_finalized_source_header_id_at_best_target.clone()
+	}
+
+	fn nonces_to_submit(&self) -> Option<RangeInclusive<MessageNonce>> {
+		self.nonces_to_submit.clone().map(|(_, nonces, _)| nonces)
+	}
+
+	fn reset_nonces_to_submit(&mut self) {
+		self.nonces_to_submit = None;
+		self.nonces_to_submit_batch = None;
+	}
+
+	fn nonces_submitted(&self) -> Option<RangeInclusive<MessageNonce>> {
+		self.nonces_submitted.clone()
+	}
+
+	fn reset_nonces_submitted(&mut self) {
+		self.nonces_submitted = None;
+	}
+}
+
+/// Run race loop until connection with target or source node is lost.
+pub async fn run<P: MessageRace, SC: SourceClient<P>, TC: TargetClient<P>>(
+	race_source: SC,
+	race_source_updated: impl FusedStream<Item = SourceClientState<P>>,
+	race_target: TC,
+	race_target_updated: impl FusedStream<Item = TargetClientState<P>>,
+	mut strategy: impl RaceStrategy<
+		P::SourceHeaderId,
+		P::TargetHeaderId,
+		P::Proof,
+		SourceNoncesRange = SC::NoncesRange,
+		ProofParameters = SC::ProofParameters,
+		TargetNoncesData = TC::TargetNoncesData,
+	>,
+) -> Result<(), FailedClient> {
+	let mut progress_context = Instant::now();
+	let mut race_state = RaceStateImpl::default();
+
+	let mut source_retry_backoff = retry_backoff();
+	let mut source_client_is_online = true;
+	let mut source_nonces_required = false;
+	let mut source_required_header = None;
+	let source_nonces = futures::future::Fuse::terminated();
+	let source_generate_proof = futures::future::Fuse::terminated();
+	let source_go_offline_future = futures::future::Fuse::terminated();
+
+	let mut target_retry_backoff = retry_backoff();
+	let mut target_client_is_online = true;
+	let mut target_best_nonces_required = false;
+	let mut target_finalized_nonces_required = false;
+	let mut target_batch_transaction = None;
+	let target_require_source_header = futures::future::Fuse::terminated();
+	let target_best_nonces = futures::future::Fuse::terminated();
+	let target_finalized_nonces = futures::future::Fuse::terminated();
+	let target_submit_proof = futures::future::Fuse::terminated();
+	let target_tx_tracker = futures::future::Fuse::terminated();
+	let target_go_offline_future = futures::future::Fuse::terminated();
+
+	futures::pin_mut!(
+		race_source_updated,
+		source_nonces,
+		source_generate_proof,
+		source_go_offline_future,
+		race_target_updated,
+		target_require_source_header,
+		target_best_nonces,
+		target_finalized_nonces,
+		target_submit_proof,
+		target_tx_tracker,
+		target_go_offline_future,
+	);
+
+	loop {
+		futures::select! {
+			// when headers ids are updated
+			source_state = race_source_updated.next() => {
+				if let Some(source_state) = source_state {
+					let is_source_state_updated = race_state.best_finalized_source_header_id_at_source.as_ref()
+						!= Some(&source_state.best_finalized_self);
+					if is_source_state_updated {
+						source_nonces_required = true;
+						race_state.best_finalized_source_header_id_at_source
+							= Some(source_state.best_finalized_self);
+					}
+				}
+			},
+			target_state = race_target_updated.next() => {
+				if let Some(target_state) = target_state {
+					let is_target_best_state_updated = race_state.best_target_header_id.as_ref()
+						!= Some(&target_state.best_self);
+
+					if is_target_best_state_updated {
+						target_best_nonces_required = true;
+						race_state.best_target_header_id = Some(target_state.best_self);
+						race_state.best_finalized_source_header_id_at_best_target
+							= target_state.best_finalized_peer_at_best_self;
+					}
+
+					let is_target_finalized_state_updated = race_state.best_finalized_target_header_id.as_ref()
+						!= Some(&target_state.best_finalized_self);
+					if  is_target_finalized_state_updated {
+						target_finalized_nonces_required = true;
+						race_state.best_finalized_target_header_id = Some(target_state.best_finalized_self);
+					}
+				}
+			},
+
+			// when nonces are updated
+			nonces = source_nonces => {
+				source_nonces_required = false;
+
+				source_client_is_online = process_future_result(
+					nonces,
+					&mut source_retry_backoff,
+					|(at_block, nonces)| {
+						tracing::debug!(
+							target: "bridge",
+							source=%P::source_name(),
+							?nonces,
+							"Received nonces"
+						);
+
+						strategy.source_nonces_updated(at_block, nonces);
+					},
+					&mut source_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving nonces from {}", P::source_name()),
+				).fail_if_connection_error(FailedClient::Source)?;
+
+				// ask for more headers if we have nonces to deliver and required headers are missing
+				source_required_header = strategy
+					.required_source_header_at_target(race_state.clone())
+					.await;
+			},
+			nonces = target_best_nonces => {
+				target_best_nonces_required = false;
+
+				target_client_is_online = process_future_result(
+					nonces,
+					&mut target_retry_backoff,
+					|(_, nonces)| {
+						tracing::debug!(
+							target: "bridge",
+							target=%P::target_name(),
+							?nonces,
+							"Received best nonces"
+						);
+
+						strategy.best_target_nonces_updated(nonces, &mut race_state);
+					},
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving best nonces from {}", P::target_name()),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+			nonces = target_finalized_nonces => {
+				target_finalized_nonces_required = false;
+
+				target_client_is_online = process_future_result(
+					nonces,
+					&mut target_retry_backoff,
+					|(_, nonces)| {
+						tracing::debug!(
+							target: "bridge",
+							target=%P::target_name(),
+							?nonces,
+							"Received finalized nonces"
+						);
+
+						strategy.finalized_target_nonces_updated(nonces, &mut race_state);
+					},
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving finalized nonces from {}", P::target_name()),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+
+			// proof generation and submission
+			maybe_batch_transaction = target_require_source_header => {
+				source_required_header = None;
+
+				target_client_is_online = process_future_result(
+					maybe_batch_transaction,
+					&mut target_retry_backoff,
+					|maybe_batch_transaction: Option<TC::BatchTransaction>| {
+						tracing::debug!(
+							target: "bridge",
+							target=%P::target_name(),
+							source=%P::source_name(),
+							batch_tx=%maybe_batch_transaction
+								.as_ref()
+								.map(|bt| format!("yes ({:?})", bt.required_header_id()))
+								.unwrap_or_else(|| "no".into()),
+							"Target client has been asked for more headers."
+						);
+
+						target_batch_transaction = maybe_batch_transaction;
+					},
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error asking for source headers at {}", P::target_name()),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+			proof = source_generate_proof => {
+				source_client_is_online = process_future_result(
+					proof,
+					&mut source_retry_backoff,
+					|(at_block, nonces_range, proof, batch_transaction)| {
+						tracing::debug!(
+							target: "bridge",
+							source=%P::source_name(),
+							?nonces_range,
+							"Received proof"
+						);
+
+						race_state.nonces_to_submit = Some((at_block, nonces_range, proof));
+						race_state.nonces_to_submit_batch = batch_transaction;
+					},
+					&mut source_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error generating proof at {}", P::source_name()),
+				).fail_if_error(FailedClient::Source).map(|_| true)?;
+			},
+			proof_submit_result = target_submit_proof => {
+				target_client_is_online = process_future_result(
+					proof_submit_result,
+					&mut target_retry_backoff,
+					|artifacts: NoncesSubmitArtifacts<TC::TransactionTracker>| {
+						tracing::debug!(
+							target: "bridge",
+							target=%P::target_name(),
+							nonces=?artifacts.nonces,
+							"Successfully submitted proof"
+						);
+
+						race_state.nonces_submitted = Some(artifacts.nonces);
+						target_tx_tracker.set(artifacts.tx_tracker.wait().fuse());
+					},
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error submitting proof {}", P::target_name()),
+				).fail_if_connection_error(FailedClient::Target)?;
+
+				// in any case - we don't need to retry submitting the same nonces again until
+				// we read nonces from the target client
+				race_state.reset_nonces_to_submit();
+				// if we have failed to submit transaction AND that is not the connection issue,
+				// then we need to read best target nonces before selecting nonces again
+				if !target_client_is_online {
+					strategy.reset_best_target_nonces();
+				}
+			},
+			target_transaction_status = target_tx_tracker => {
+				match (target_transaction_status, race_state.nonces_submitted.as_ref()) {
+					(TrackedTransactionStatus::Finalized(at_block), Some(nonces_submitted)) => {
+						// our transaction has been mined, but was it successful or not? let's check the best
+						// nonce at the target node.
+						let _ = race_target.nonces(at_block, false)
+							.await
+							.map_err(|e| format!("failed to read nonces from target node: {e:?}"))
+							.and_then(|(_, nonces_at_target)| {
+								if nonces_at_target.latest_nonce < *nonces_submitted.end() {
+									Err(format!(
+										"best nonce at target after tx is {:?} and we've submitted {:?}",
+										nonces_at_target.latest_nonce,
+										nonces_submitted.end(),
+									))
+								} else {
+									Ok(())
+								}
+							})
+							.map_err(|e| {
+								tracing::error!(
+									target: "bridge",
+									error=%e,
+									source=%P::source_name(),
+									target=%P::target_name(),
+									"Source -> target race transaction failed"
+								);
+
+								race_state.reset_nonces_submitted();
+							});
+					},
+					(TrackedTransactionStatus::Lost, _) => {
+						tracing::warn!(
+							target: "bridge",
+							source=%P::source_name(),
+							target=%P::target_name(),
+							state=?race_state,
+							?strategy,
+							"Source -> target race transaction has been lost."
+						);
+
+						race_state.reset_nonces_submitted();
+					},
+					_ => (),
+				}
+			},
+
+			// when we're ready to retry request
+			_ = source_go_offline_future => {
+				source_client_is_online = true;
+			},
+			_ = target_go_offline_future => {
+				target_client_is_online = true;
+			},
+		}
+
+		progress_context = print_race_progress::<P, _>(progress_context, &strategy);
+
+		if source_client_is_online {
+			source_client_is_online = false;
+
+			// if we've started to submit batch transaction, let's prioritize it
+			//
+			// we're using `take` here, because we don't need batch transaction (i.e. some
+			// underlying finality proof) anymore for our future calls - we were unable to
+			// use it for our current state, so why would we need to keep an obsolete proof
+			// for the future?
+			let target_batch_transaction = target_batch_transaction.take();
+			let expected_race_state =
+				if let Some(ref target_batch_transaction) = target_batch_transaction {
+					// when selecting nonces for the batch transaction, we assume that the required
+					// source header is already at the target chain
+					let required_source_header_at_target =
+						target_batch_transaction.required_header_id();
+					let mut expected_race_state = race_state.clone();
+					expected_race_state.best_finalized_source_header_id_at_best_target =
+						Some(required_source_header_at_target);
+					expected_race_state
+				} else {
+					race_state.clone()
+				};
+
+			let nonces_to_deliver = select_nonces_to_deliver(expected_race_state, &strategy).await;
+			let best_at_source = strategy.best_at_source();
+
+			if let Some((at_block, nonces_range, proof_parameters)) = nonces_to_deliver {
+				tracing::debug!(
+					target: "bridge",
+					source=P::source_name(),
+					?nonces_range,
+					?at_block,
+					"Asking to prove"
+				);
+
+				source_generate_proof.set(
+					race_source
+						.generate_proof(at_block, nonces_range, proof_parameters)
+						.and_then(|(at_source_block, nonces, proof)| async {
+							Ok((at_source_block, nonces, proof, target_batch_transaction))
+						})
+						.fuse(),
+				);
+			} else if let (true, Some(best_at_source)) = (source_nonces_required, best_at_source) {
+				tracing::debug!(target: "bridge", source=%P::source_name(), "Asking about message nonces");
+				let at_block = race_state
+					.best_finalized_source_header_id_at_source
+					.as_ref()
+					.expect(
+						"source_nonces_required is only true when \
+						best_finalized_source_header_id_at_source is Some; qed",
+					)
+					.clone();
+				source_nonces.set(race_source.nonces(at_block, best_at_source).fuse());
+			} else {
+				source_client_is_online = true;
+			}
+		}
+
+		if target_client_is_online {
+			target_client_is_online = false;
+
+			if let Some((at_block, nonces_range, proof)) = race_state.nonces_to_submit.as_ref() {
+				tracing::debug!(
+					target: "bridge",
+					target=%P::target_name(),
+					?nonces_range,
+					"Going to submit proof of messages in range to node{}",
+					race_state.nonces_to_submit_batch.as_ref().map(|tx| format!(
+						". This transaction is batched with sending the proof for header {:?}.",
+						tx.required_header_id())
+					).unwrap_or_default(),
+				);
+
+				target_submit_proof.set(
+					race_target
+						.submit_proof(
+							race_state.nonces_to_submit_batch.clone(),
+							at_block.clone(),
+							nonces_range.clone(),
+							proof.clone(),
+						)
+						.fuse(),
+				);
+			} else if let Some(source_required_header) = source_required_header.clone() {
+				tracing::debug!(
+					target: "bridge",
+					source=%P::source_name(),
+					target=%P::target_name(),
+					?source_required_header,
+					"Going to require header"
+				);
+				target_require_source_header
+					.set(race_target.require_source_header(source_required_header).fuse());
+			} else if target_best_nonces_required {
+				tracing::debug!(target: "bridge", target=%P::target_name(), "Asking about best message nonces");
+				let at_block = race_state
+					.best_target_header_id
+					.as_ref()
+					.expect("target_best_nonces_required is only true when best_target_header_id is Some; qed")
+					.clone();
+				target_best_nonces.set(race_target.nonces(at_block, false).fuse());
+			} else if target_finalized_nonces_required {
+				tracing::debug!(target: "bridge", target=%P::target_name(), "Asking about finalized message nonces");
+				let at_block = race_state
+					.best_finalized_target_header_id
+					.as_ref()
+					.expect(
+						"target_finalized_nonces_required is only true when \
+						best_finalized_target_header_id is Some; qed",
+					)
+					.clone();
+				target_finalized_nonces.set(race_target.nonces(at_block, true).fuse());
+			} else {
+				target_client_is_online = true;
+			}
+		}
+	}
+}
+
+/// Print race progress.
+fn print_race_progress<P, S>(prev_time: Instant, strategy: &S) -> Instant
+where
+	P: MessageRace,
+	S: RaceStrategy<P::SourceHeaderId, P::TargetHeaderId, P::Proof>,
+{
+	let now_time = Instant::now();
+
+	let need_update = now_time.saturating_duration_since(prev_time) > Duration::from_secs(10);
+	if !need_update {
+		return prev_time;
+	}
+
+	let now_best_nonce_at_source = strategy.best_at_source();
+	let now_best_nonce_at_target = strategy.best_at_target();
+	tracing::info!(
+		target: "bridge",
+		source=%P::source_name(),
+		target=%P::target_name(),
+		?now_best_nonce_at_target,
+		?now_best_nonce_at_source,
+		"Synced nonces in source -> target race"
+
+	);
+	now_time
+}
+
+async fn select_nonces_to_deliver<SourceHeaderId, TargetHeaderId, Proof, Strategy>(
+	race_state: impl RaceState<SourceHeaderId, TargetHeaderId>,
+	strategy: &Strategy,
+) -> Option<(SourceHeaderId, RangeInclusive<MessageNonce>, Strategy::ProofParameters)>
+where
+	SourceHeaderId: Clone,
+	Strategy: RaceStrategy<SourceHeaderId, TargetHeaderId, Proof>,
+{
+	let best_finalized_source_header_id_at_best_target =
+		race_state.best_finalized_source_header_id_at_best_target()?;
+	strategy
+		.select_nonces_to_deliver(race_state)
+		.await
+		.map(|(nonces_range, proof_parameters)| {
+			(best_finalized_source_header_id_at_best_target, nonces_range, proof_parameters)
+		})
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::message_race_strategy::BasicStrategy;
+	use relay_utils::HeaderId;
+
+	#[async_std::test]
+	async fn proof_is_generated_at_best_block_known_to_target_node() {
+		const GENERATED_AT: u64 = 6;
+		const BEST_AT_SOURCE: u64 = 10;
+		const BEST_AT_TARGET: u64 = 8;
+
+		// target node only knows about source' BEST_AT_TARGET block
+		// source node has BEST_AT_SOURCE > BEST_AT_TARGET block
+		let mut race_state = RaceStateImpl::<_, _, (), ()> {
+			best_finalized_source_header_id_at_source: Some(HeaderId(
+				BEST_AT_SOURCE,
+				BEST_AT_SOURCE,
+			)),
+			best_finalized_source_header_id_at_best_target: Some(HeaderId(
+				BEST_AT_TARGET,
+				BEST_AT_TARGET,
+			)),
+			best_target_header_id: Some(HeaderId(0, 0)),
+			best_finalized_target_header_id: Some(HeaderId(0, 0)),
+			nonces_to_submit: None,
+			nonces_to_submit_batch: None,
+			nonces_submitted: None,
+		};
+
+		// we have some nonces to deliver and they're generated at GENERATED_AT < BEST_AT_SOURCE
+		let mut strategy = BasicStrategy::<_, _, _, _, _, ()>::new();
+		strategy.source_nonces_updated(
+			HeaderId(GENERATED_AT, GENERATED_AT),
+			SourceClientNonces { new_nonces: 0..=10, confirmed_nonce: None },
+		);
+		strategy.best_target_nonces_updated(
+			TargetClientNonces { latest_nonce: 5u64, nonces_data: () },
+			&mut race_state,
+		);
+
+		// the proof will be generated on source, but using BEST_AT_TARGET block
+		assert_eq!(
+			select_nonces_to_deliver(race_state, &strategy).await,
+			Some((HeaderId(BEST_AT_TARGET, BEST_AT_TARGET), 6..=10, (),))
+		);
+	}
+}

pezbridges/relays/messages/src/message_race_receiving.rs

@@ -0,0 +1,272 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+//! Message receiving race delivers proof-of-messages-delivery from "lane.target" to "lane.source".
+
+use crate::{
+	message_lane::{MessageLane, SourceHeaderIdOf, TargetHeaderIdOf},
+	message_lane_loop::{
+		NoncesSubmitArtifacts, SourceClient as MessageLaneSourceClient, SourceClientState,
+		TargetClient as MessageLaneTargetClient, TargetClientState,
+	},
+	message_race_loop::{
+		MessageRace, NoncesRange, SourceClient, SourceClientNonces, TargetClient,
+		TargetClientNonces,
+	},
+	message_race_strategy::BasicStrategy,
+	metrics::MessageLaneLoopMetrics,
+};
+
+use async_trait::async_trait;
+use bp_messages::MessageNonce;
+use futures::stream::FusedStream;
+use relay_utils::{FailedClient, TrackedTransactionStatus, TransactionTracker};
+use std::{marker::PhantomData, ops::RangeInclusive};
+
+/// Message receiving confirmations delivery strategy.
+type ReceivingConfirmationsBasicStrategy<P> = BasicStrategy<
+	<P as MessageLane>::TargetHeaderNumber,
+	<P as MessageLane>::TargetHeaderHash,
+	<P as MessageLane>::SourceHeaderNumber,
+	<P as MessageLane>::SourceHeaderHash,
+	RangeInclusive<MessageNonce>,
+	<P as MessageLane>::MessagesReceivingProof,
+>;
+
+/// Run receiving confirmations race.
+pub async fn run<P: MessageLane>(
+	source_client: impl MessageLaneSourceClient<P>,
+	source_state_updates: impl FusedStream<Item = SourceClientState<P>>,
+	target_client: impl MessageLaneTargetClient<P>,
+	target_state_updates: impl FusedStream<Item = TargetClientState<P>>,
+	metrics_msg: Option<MessageLaneLoopMetrics>,
+) -> Result<(), FailedClient> {
+	crate::message_race_loop::run(
+		ReceivingConfirmationsRaceSource {
+			client: target_client,
+			metrics_msg: metrics_msg.clone(),
+			_phantom: Default::default(),
+		},
+		target_state_updates,
+		ReceivingConfirmationsRaceTarget {
+			client: source_client,
+			metrics_msg,
+			_phantom: Default::default(),
+		},
+		source_state_updates,
+		ReceivingConfirmationsBasicStrategy::<P>::new(),
+	)
+	.await
+}
+
+/// Relay messages delivery confirmation.
+pub async fn relay_messages_delivery_confirmation<P: MessageLane>(
+	source_client: impl MessageLaneSourceClient<P>,
+	target_client: impl MessageLaneTargetClient<P>,
+	at: TargetHeaderIdOf<P>,
+) -> Result<(), ()> {
+	// prepare messages delivery proof
+	let (at, proof) = target_client.prove_messages_receiving(at.clone()).await.map_err(|e| {
+		tracing::error!(
+			target: "bridge",
+			error=?e,
+			?at,
+			"Failed to generate messages delivery proof",
+		);
+	})?;
+	// submit messages delivery proof to the source node
+	let tx_tracker =
+		source_client
+			.submit_messages_receiving_proof(None, at, proof)
+			.await
+			.map_err(|e| {
+				tracing::error!(
+					target: "bridge",
+					error=?e,
+					"Failed to submit messages delivery proof"
+				);
+			})?;
+
+	match tx_tracker.wait().await {
+		TrackedTransactionStatus::Finalized(_) => Ok(()),
+		TrackedTransactionStatus::Lost => {
+			tracing::error!(target: "bridge", "Transaction with messages delivery proof is considered lost");
+			Err(())
+		},
+	}
+}
+
+/// Messages receiving confirmations race.
+struct ReceivingConfirmationsRace<P>(std::marker::PhantomData<P>);
+
+impl<P: MessageLane> MessageRace for ReceivingConfirmationsRace<P> {
+	type SourceHeaderId = TargetHeaderIdOf<P>;
+	type TargetHeaderId = SourceHeaderIdOf<P>;
+
+	type MessageNonce = MessageNonce;
+	type Proof = P::MessagesReceivingProof;
+
+	fn source_name() -> String {
+		format!("{}::ReceivingConfirmationsDelivery", P::TARGET_NAME)
+	}
+
+	fn target_name() -> String {
+		format!("{}::ReceivingConfirmationsDelivery", P::SOURCE_NAME)
+	}
+}
+
+/// Message receiving confirmations race source, which is a target of the lane.
+struct ReceivingConfirmationsRaceSource<P: MessageLane, C> {
+	client: C,
+	metrics_msg: Option<MessageLaneLoopMetrics>,
+	_phantom: PhantomData<P>,
+}
+
+#[async_trait]
+impl<P, C> SourceClient<ReceivingConfirmationsRace<P>> for ReceivingConfirmationsRaceSource<P, C>
+where
+	P: MessageLane,
+	C: MessageLaneTargetClient<P>,
+{
+	type Error = C::Error;
+	type NoncesRange = RangeInclusive<MessageNonce>;
+	type ProofParameters = ();
+
+	async fn nonces(
+		&self,
+		at_block: TargetHeaderIdOf<P>,
+		prev_latest_nonce: MessageNonce,
+	) -> Result<(TargetHeaderIdOf<P>, SourceClientNonces<Self::NoncesRange>), Self::Error> {
+		let (at_block, latest_received_nonce) = self.client.latest_received_nonce(at_block).await?;
+		if let Some(metrics_msg) = self.metrics_msg.as_ref() {
+			metrics_msg.update_target_latest_received_nonce(latest_received_nonce);
+		}
+		Ok((
+			at_block,
+			SourceClientNonces {
+				new_nonces: prev_latest_nonce + 1..=latest_received_nonce,
+				confirmed_nonce: None,
+			},
+		))
+	}
+
+	#[allow(clippy::unit_arg)]
+	async fn generate_proof(
+		&self,
+		at_block: TargetHeaderIdOf<P>,
+		nonces: RangeInclusive<MessageNonce>,
+		_proof_parameters: Self::ProofParameters,
+	) -> Result<
+		(TargetHeaderIdOf<P>, RangeInclusive<MessageNonce>, P::MessagesReceivingProof),
+		Self::Error,
+	> {
+		self.client
+			.prove_messages_receiving(at_block)
+			.await
+			.map(|(at_block, proof)| (at_block, nonces, proof))
+	}
+}
+
+/// Message receiving confirmations race target, which is a source of the lane.
+struct ReceivingConfirmationsRaceTarget<P: MessageLane, C> {
+	client: C,
+	metrics_msg: Option<MessageLaneLoopMetrics>,
+	_phantom: PhantomData<P>,
+}
+
+#[async_trait]
+impl<P, C> TargetClient<ReceivingConfirmationsRace<P>> for ReceivingConfirmationsRaceTarget<P, C>
+where
+	P: MessageLane,
+	C: MessageLaneSourceClient<P>,
+{
+	type Error = C::Error;
+	type TargetNoncesData = ();
+	type BatchTransaction = C::BatchTransaction;
+	type TransactionTracker = C::TransactionTracker;
+
+	async fn require_source_header(
+		&self,
+		id: TargetHeaderIdOf<P>,
+	) -> Result<Option<C::BatchTransaction>, Self::Error> {
+		self.client.require_target_header_on_source(id).await
+	}
+
+	async fn nonces(
+		&self,
+		at_block: SourceHeaderIdOf<P>,
+		update_metrics: bool,
+	) -> Result<(SourceHeaderIdOf<P>, TargetClientNonces<()>), Self::Error> {
+		let (at_block, latest_confirmed_nonce) =
+			self.client.latest_confirmed_received_nonce(at_block).await?;
+		if update_metrics {
+			if let Some(metrics_msg) = self.metrics_msg.as_ref() {
+				metrics_msg.update_source_latest_confirmed_nonce(latest_confirmed_nonce);
+			}
+		}
+		Ok((at_block, TargetClientNonces { latest_nonce: latest_confirmed_nonce, nonces_data: () }))
+	}
+
+	async fn submit_proof(
+		&self,
+		maybe_batch_tx: Option<Self::BatchTransaction>,
+		generated_at_block: TargetHeaderIdOf<P>,
+		nonces: RangeInclusive<MessageNonce>,
+		proof: P::MessagesReceivingProof,
+	) -> Result<NoncesSubmitArtifacts<Self::TransactionTracker>, Self::Error> {
+		let tx_tracker = self
+			.client
+			.submit_messages_receiving_proof(maybe_batch_tx, generated_at_block, proof)
+			.await?;
+		Ok(NoncesSubmitArtifacts { nonces, tx_tracker })
+	}
+}
+
+impl NoncesRange for RangeInclusive<MessageNonce> {
+	fn begin(&self) -> MessageNonce {
+		*RangeInclusive::<MessageNonce>::start(self)
+	}
+
+	fn end(&self) -> MessageNonce {
+		*RangeInclusive::<MessageNonce>::end(self)
+	}
+
+	fn greater_than(self, nonce: MessageNonce) -> Option<Self> {
+		let next_nonce = nonce + 1;
+		let end = *self.end();
+		if next_nonce > end {
+			None
+		} else {
+			Some(std::cmp::max(self.begin(), next_nonce)..=end)
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn range_inclusive_works_as_nonces_range() {
+		let range = 20..=30;
+
+		assert_eq!(NoncesRange::begin(&range), 20);
+		assert_eq!(NoncesRange::end(&range), 30);
+		assert_eq!(range.clone().greater_than(10), Some(20..=30));
+		assert_eq!(range.clone().greater_than(19), Some(20..=30));
+		assert_eq!(range.clone().greater_than(20), Some(21..=30));
+		assert_eq!(range.clone().greater_than(25), Some(26..=30));
+		assert_eq!(range.clone().greater_than(29), Some(30..=30));
+		assert_eq!(range.greater_than(30), None);
+	}
+}

pezbridges/relays/messages/src/message_race_strategy.rs

@@ -0,0 +1,618 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+//! Basic delivery strategy. The strategy selects nonces if:
+//!
+//! 1) there are more nonces on the source side than on the target side;
+//! 2) new nonces may be proved to target node (i.e. they have appeared at the block, which is known
+//!    to the target node).
+
+use crate::message_race_loop::{
+	NoncesRange, RaceState, RaceStrategy, SourceClientNonces, TargetClientNonces,
+};
+
+use async_trait::async_trait;
+use bp_messages::MessageNonce;
+use relay_utils::HeaderId;
+use std::{collections::VecDeque, fmt::Debug, marker::PhantomData, ops::RangeInclusive};
+
+/// Queue of nonces known to the source node.
+pub type SourceRangesQueue<SourceHeaderHash, SourceHeaderNumber, SourceNoncesRange> =
+	VecDeque<(HeaderId<SourceHeaderHash, SourceHeaderNumber>, SourceNoncesRange)>;
+
+/// Nonces delivery strategy.
+#[derive(Debug)]
+pub struct BasicStrategy<
+	SourceHeaderNumber,
+	SourceHeaderHash,
+	TargetHeaderNumber,
+	TargetHeaderHash,
+	SourceNoncesRange,
+	Proof,
+> {
+	/// All queued nonces.
+	///
+	/// The queue may contain already delivered nonces. We only remove entries from this
+	/// queue after corresponding nonces are finalized by the target chain.
+	source_queue: SourceRangesQueue<SourceHeaderHash, SourceHeaderNumber, SourceNoncesRange>,
+	/// The best nonce known to target node at its best block. `None` if it has not been received
+	/// yet.
+	best_target_nonce: Option<MessageNonce>,
+	/// Unused generic types dump.
+	_phantom: PhantomData<(TargetHeaderNumber, TargetHeaderHash, Proof)>,
+}
+
+impl<
+		SourceHeaderNumber,
+		SourceHeaderHash,
+		TargetHeaderNumber,
+		TargetHeaderHash,
+		SourceNoncesRange,
+		Proof,
+	>
+	BasicStrategy<
+		SourceHeaderNumber,
+		SourceHeaderHash,
+		TargetHeaderNumber,
+		TargetHeaderHash,
+		SourceNoncesRange,
+		Proof,
+	>
+where
+	SourceHeaderHash: Clone,
+	SourceHeaderNumber: Clone + Ord,
+	SourceNoncesRange: NoncesRange,
+{
+	/// Create new delivery strategy.
+	pub fn new() -> Self {
+		BasicStrategy {
+			source_queue: VecDeque::new(),
+			best_target_nonce: None,
+			_phantom: Default::default(),
+		}
+	}
+
+	/// Reference to source queue.
+	pub(crate) fn source_queue(
+		&self,
+	) -> &VecDeque<(HeaderId<SourceHeaderHash, SourceHeaderNumber>, SourceNoncesRange)> {
+		&self.source_queue
+	}
+
+	/// Mutable reference to source queue to use in tests.
+	#[cfg(test)]
+	pub(crate) fn source_queue_mut(
+		&mut self,
+	) -> &mut VecDeque<(HeaderId<SourceHeaderHash, SourceHeaderNumber>, SourceNoncesRange)> {
+		&mut self.source_queue
+	}
+
+	/// Returns indices of source queue entries, which may be delivered to the target node.
+	///
+	/// The function may skip some nonces from the queue front if nonces from this entry are
+	/// already available at the **best** target block. After this block is finalized, the entry
+	/// will be removed from the queue.
+	///
+	/// All entries before and including the range end index, are guaranteed to be witnessed
+	/// at source blocks that are known to be finalized at the target node.
+	///
+	/// Returns `None` if no entries may be delivered.
+	pub fn available_source_queue_indices<
+		RS: RaceState<
+			HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+			HeaderId<TargetHeaderHash, TargetHeaderNumber>,
+		>,
+	>(
+		&self,
+		race_state: RS,
+	) -> Option<RangeInclusive<usize>> {
+		// if we do not know best nonce at target node, we can't select anything
+		let best_target_nonce = self.best_target_nonce?;
+
+		// if we have already selected nonces that we want to submit, do nothing
+		if race_state.nonces_to_submit().is_some() {
+			return None;
+		}
+
+		// if we already submitted some nonces, do nothing
+		if race_state.nonces_submitted().is_some() {
+			return None;
+		}
+
+		// find first entry that may be delivered to the target node
+		let begin_index = self
+			.source_queue
+			.iter()
+			.enumerate()
+			.skip_while(|(_, (_, nonces))| nonces.end() <= best_target_nonce)
+			.map(|(index, _)| index)
+			.next()?;
+
+		// 1) we want to deliver all nonces, starting from `target_nonce + 1`
+		// 2) we can't deliver new nonce until header, that has emitted this nonce, is finalized
+		// by target client
+		// 3) selector is used for more complicated logic
+		//
+		// => let's first select range of entries inside deque that are already finalized at
+		// the target client and pass this range to the selector
+		let best_header_at_target = race_state.best_finalized_source_header_id_at_best_target()?;
+		let end_index = self
+			.source_queue
+			.iter()
+			.enumerate()
+			.skip(begin_index)
+			.take_while(|(_, (queued_at, _))| queued_at.0 <= best_header_at_target.0)
+			.map(|(index, _)| index)
+			.last()?;
+
+		Some(begin_index..=end_index)
+	}
+
+	/// Remove all nonces that are less than or equal to given nonce from the source queue.
+	fn remove_le_nonces_from_source_queue(&mut self, nonce: MessageNonce) {
+		while let Some((queued_at, queued_range)) = self.source_queue.pop_front() {
+			if let Some(range_to_requeue) = queued_range.greater_than(nonce) {
+				self.source_queue.push_front((queued_at, range_to_requeue));
+				break;
+			}
+		}
+	}
+}
+
+#[async_trait]
+impl<
+		SourceHeaderNumber,
+		SourceHeaderHash,
+		TargetHeaderNumber,
+		TargetHeaderHash,
+		SourceNoncesRange,
+		Proof,
+	>
+	RaceStrategy<
+		HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+		HeaderId<TargetHeaderHash, TargetHeaderNumber>,
+		Proof,
+	>
+	for BasicStrategy<
+		SourceHeaderNumber,
+		SourceHeaderHash,
+		TargetHeaderNumber,
+		TargetHeaderHash,
+		SourceNoncesRange,
+		Proof,
+	>
+where
+	SourceHeaderHash: Clone + Debug + Send + Sync,
+	SourceHeaderNumber: Clone + Ord + Debug + Send + Sync,
+	SourceNoncesRange: NoncesRange + Debug + Send + Sync,
+	TargetHeaderHash: Debug + Send + Sync,
+	TargetHeaderNumber: Debug + Send + Sync,
+	Proof: Debug + Send + Sync,
+{
+	type SourceNoncesRange = SourceNoncesRange;
+	type ProofParameters = ();
+	type TargetNoncesData = ();
+
+	async fn required_source_header_at_target<
+		RS: RaceState<
+			HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+			HeaderId<TargetHeaderHash, TargetHeaderNumber>,
+		>,
+	>(
+		&self,
+		race_state: RS,
+	) -> Option<HeaderId<SourceHeaderHash, SourceHeaderNumber>> {
+		let current_best = race_state.best_finalized_source_header_id_at_best_target()?;
+		self.source_queue
+			.back()
+			.and_then(|(h, _)| if h.0 > current_best.0 { Some(h.clone()) } else { None })
+	}
+
+	fn best_at_source(&self) -> Option<MessageNonce> {
+		let best_in_queue = self.source_queue.back().map(|(_, range)| range.end());
+		match (best_in_queue, self.best_target_nonce) {
+			(Some(best_in_queue), Some(best_target_nonce)) if best_in_queue > best_target_nonce =>
+				Some(best_in_queue),
+			(_, Some(best_target_nonce)) => Some(best_target_nonce),
+			(_, None) => None,
+		}
+	}
+
+	fn best_at_target(&self) -> Option<MessageNonce> {
+		self.best_target_nonce
+	}
+
+	fn source_nonces_updated(
+		&mut self,
+		at_block: HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+		nonces: SourceClientNonces<SourceNoncesRange>,
+	) {
+		let best_in_queue = self
+			.source_queue
+			.back()
+			.map(|(_, range)| range.end())
+			.or(self.best_target_nonce)
+			.unwrap_or_default();
+		self.source_queue.extend(
+			nonces
+				.new_nonces
+				.greater_than(best_in_queue)
+				.into_iter()
+				.map(move |range| (at_block.clone(), range)),
+		)
+	}
+
+	fn reset_best_target_nonces(&mut self) {
+		self.best_target_nonce = None;
+	}
+
+	fn best_target_nonces_updated<
+		RS: RaceState<
+			HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+			HeaderId<TargetHeaderHash, TargetHeaderNumber>,
+		>,
+	>(
+		&mut self,
+		nonces: TargetClientNonces<()>,
+		race_state: &mut RS,
+	) {
+		let nonce = nonces.latest_nonce;
+
+		// if **some** of nonces that we have selected to submit already present at the
+		// target chain => select new nonces
+		let need_to_select_new_nonces = race_state
+			.nonces_to_submit()
+			.map(|nonces| nonce >= *nonces.start())
+			.unwrap_or(false);
+		if need_to_select_new_nonces {
+			tracing::trace!(
+				target: "bridge",
+				%nonce,
+				nonces_to_submit=?race_state.nonces_to_submit(),
+				"Latest nonce at target. Clearing nonces to submit"
+			);
+
+			race_state.reset_nonces_to_submit();
+		}
+
+		// if **some** of nonces that we have submitted already present at the
+		// target chain => select new nonces
+		let need_new_nonces_to_submit = race_state
+			.nonces_submitted()
+			.map(|nonces| nonce >= *nonces.start())
+			.unwrap_or(false);
+		if need_new_nonces_to_submit {
+			tracing::trace!(
+				target: "bridge",
+				%nonce,
+				nonces_submitted=?race_state.nonces_submitted(),
+				"Latest nonce at target. Clearing submitted nonces"
+			);
+
+			race_state.reset_nonces_submitted();
+		}
+
+		self.best_target_nonce = Some(nonce);
+	}
+
+	fn finalized_target_nonces_updated<
+		RS: RaceState<
+			HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+			HeaderId<TargetHeaderHash, TargetHeaderNumber>,
+		>,
+	>(
+		&mut self,
+		nonces: TargetClientNonces<()>,
+		_race_state: &mut RS,
+	) {
+		self.remove_le_nonces_from_source_queue(nonces.latest_nonce);
+		self.best_target_nonce = Some(std::cmp::max(
+			self.best_target_nonce.unwrap_or(nonces.latest_nonce),
+			nonces.latest_nonce,
+		));
+	}
+
+	async fn select_nonces_to_deliver<
+		RS: RaceState<
+			HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+			HeaderId<TargetHeaderHash, TargetHeaderNumber>,
+		>,
+	>(
+		&self,
+		race_state: RS,
+	) -> Option<(RangeInclusive<MessageNonce>, Self::ProofParameters)> {
+		let available_indices = self.available_source_queue_indices(race_state)?;
+		let range_begin = std::cmp::max(
+			self.best_target_nonce? + 1,
+			self.source_queue[*available_indices.start()].1.begin(),
+		);
+		let range_end = self.source_queue[*available_indices.end()].1.end();
+		Some((range_begin..=range_end, ()))
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::{
+		message_lane::{MessageLane, SourceHeaderIdOf, TargetHeaderIdOf},
+		message_lane_loop::tests::{
+			header_id, TestMessageLane, TestMessagesProof, TestSourceHeaderHash,
+			TestSourceHeaderNumber,
+		},
+		message_race_loop::RaceStateImpl,
+	};
+
+	type SourceNoncesRange = RangeInclusive<MessageNonce>;
+
+	type TestRaceStateImpl = RaceStateImpl<
+		SourceHeaderIdOf<TestMessageLane>,
+		TargetHeaderIdOf<TestMessageLane>,
+		TestMessagesProof,
+		(),
+	>;
+
+	type BasicStrategy<P> = super::BasicStrategy<
+		<P as MessageLane>::SourceHeaderNumber,
+		<P as MessageLane>::SourceHeaderHash,
+		<P as MessageLane>::TargetHeaderNumber,
+		<P as MessageLane>::TargetHeaderHash,
+		SourceNoncesRange,
+		<P as MessageLane>::MessagesProof,
+	>;
+
+	fn source_nonces(new_nonces: SourceNoncesRange) -> SourceClientNonces<SourceNoncesRange> {
+		SourceClientNonces { new_nonces, confirmed_nonce: None }
+	}
+
+	fn target_nonces(latest_nonce: MessageNonce) -> TargetClientNonces<()> {
+		TargetClientNonces { latest_nonce, nonces_data: () }
+	}
+
+	#[test]
+	fn best_at_source_is_never_lower_than_target_nonce() {
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		assert_eq!(strategy.best_at_source(), None);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=5));
+		assert_eq!(strategy.best_at_source(), None);
+		strategy.best_target_nonces_updated(target_nonces(10), &mut TestRaceStateImpl::default());
+		assert_eq!(strategy.source_queue, vec![(header_id(1), 1..=5)]);
+		assert_eq!(strategy.best_at_source(), Some(10));
+	}
+
+	#[test]
+	fn source_nonce_is_never_lower_than_known_target_nonce() {
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.best_target_nonces_updated(target_nonces(10), &mut TestRaceStateImpl::default());
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=5));
+		assert_eq!(strategy.source_queue, vec![]);
+	}
+
+	#[test]
+	fn source_nonce_is_never_lower_than_latest_known_source_nonce() {
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=5));
+		strategy.source_nonces_updated(header_id(2), source_nonces(1..=3));
+		strategy.source_nonces_updated(header_id(2), source_nonces(1..=5));
+		assert_eq!(strategy.source_queue, vec![(header_id(1), 1..=5)]);
+	}
+
+	#[test]
+	fn updated_target_nonce_removes_queued_entries() {
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=5));
+		strategy.source_nonces_updated(header_id(2), source_nonces(6..=10));
+		strategy.source_nonces_updated(header_id(3), source_nonces(11..=15));
+		strategy.source_nonces_updated(header_id(4), source_nonces(16..=20));
+		strategy
+			.finalized_target_nonces_updated(target_nonces(15), &mut TestRaceStateImpl::default());
+		assert_eq!(strategy.source_queue, vec![(header_id(4), 16..=20)]);
+		strategy
+			.finalized_target_nonces_updated(target_nonces(17), &mut TestRaceStateImpl::default());
+		assert_eq!(strategy.source_queue, vec![(header_id(4), 18..=20)]);
+	}
+
+	#[test]
+	fn selected_nonces_are_dropped_on_target_nonce_update() {
+		let mut state = TestRaceStateImpl::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		state.nonces_to_submit = Some((header_id(1), 5..=10, (5..=10, None)));
+		// we are going to submit 5..=10, so having latest nonce 4 at target is fine
+		strategy.best_target_nonces_updated(target_nonces(4), &mut state);
+		assert!(state.nonces_to_submit.is_some());
+		// any nonce larger than 4 invalidates the `nonces_to_submit`
+		for nonce in 5..=11 {
+			state.nonces_to_submit = Some((header_id(1), 5..=10, (5..=10, None)));
+			strategy.best_target_nonces_updated(target_nonces(nonce), &mut state);
+			assert!(state.nonces_to_submit.is_none());
+		}
+	}
+
+	#[test]
+	fn submitted_nonces_are_dropped_on_target_nonce_update() {
+		let mut state = TestRaceStateImpl::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		state.nonces_submitted = Some(5..=10);
+		// we have submitted 5..=10, so having latest nonce 4 at target is fine
+		strategy.best_target_nonces_updated(target_nonces(4), &mut state);
+		assert!(state.nonces_submitted.is_some());
+		// any nonce larger than 4 invalidates the `nonces_submitted`
+		for nonce in 5..=11 {
+			state.nonces_submitted = Some(5..=10);
+			strategy.best_target_nonces_updated(target_nonces(nonce), &mut state);
+			assert!(state.nonces_submitted.is_none());
+		}
+	}
+
+	#[async_std::test]
+	async fn nothing_is_selected_if_something_is_already_selected() {
+		let mut state = TestRaceStateImpl::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		state.nonces_to_submit = Some((header_id(1), 1..=10, (1..=10, None)));
+		strategy.best_target_nonces_updated(target_nonces(0), &mut state);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=10));
+		assert_eq!(strategy.select_nonces_to_deliver(state.clone()).await, None);
+	}
+
+	#[async_std::test]
+	async fn nothing_is_selected_if_something_is_already_submitted() {
+		let mut state = TestRaceStateImpl::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		state.nonces_submitted = Some(1..=10);
+		strategy.best_target_nonces_updated(target_nonces(0), &mut state);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=10));
+		assert_eq!(strategy.select_nonces_to_deliver(state.clone()).await, None);
+	}
+
+	#[async_std::test]
+	async fn select_nonces_to_deliver_works() {
+		let mut state = TestRaceStateImpl::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.best_target_nonces_updated(target_nonces(0), &mut state);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=1));
+		strategy.source_nonces_updated(header_id(2), source_nonces(2..=2));
+		strategy.source_nonces_updated(header_id(3), source_nonces(3..=6));
+		strategy.source_nonces_updated(header_id(5), source_nonces(7..=8));
+
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(4));
+		assert_eq!(strategy.select_nonces_to_deliver(state.clone()).await, Some((1..=6, ())));
+		strategy.best_target_nonces_updated(target_nonces(6), &mut state);
+		assert_eq!(strategy.select_nonces_to_deliver(state.clone()).await, None);
+
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(5));
+		assert_eq!(strategy.select_nonces_to_deliver(state.clone()).await, Some((7..=8, ())));
+		strategy.best_target_nonces_updated(target_nonces(8), &mut state);
+		assert_eq!(strategy.select_nonces_to_deliver(state.clone()).await, None);
+	}
+
+	#[test]
+	fn available_source_queue_indices_works() {
+		let mut state = TestRaceStateImpl::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.best_target_nonces_updated(target_nonces(0), &mut state);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=3));
+		strategy.source_nonces_updated(header_id(2), source_nonces(4..=6));
+		strategy.source_nonces_updated(header_id(3), source_nonces(7..=9));
+
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(0));
+		assert_eq!(strategy.available_source_queue_indices(state.clone()), None);
+
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(1));
+		assert_eq!(strategy.available_source_queue_indices(state.clone()), Some(0..=0));
+
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(2));
+		assert_eq!(strategy.available_source_queue_indices(state.clone()), Some(0..=1));
+
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(3));
+		assert_eq!(strategy.available_source_queue_indices(state.clone()), Some(0..=2));
+
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(4));
+		assert_eq!(strategy.available_source_queue_indices(state), Some(0..=2));
+	}
+
+	#[test]
+	fn remove_le_nonces_from_source_queue_works() {
+		let mut state = TestRaceStateImpl::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.best_target_nonces_updated(target_nonces(0), &mut state);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=3));
+		strategy.source_nonces_updated(header_id(2), source_nonces(4..=6));
+		strategy.source_nonces_updated(header_id(3), source_nonces(7..=9));
+
+		fn source_queue_nonces(
+			source_queue: &SourceRangesQueue<
+				TestSourceHeaderHash,
+				TestSourceHeaderNumber,
+				SourceNoncesRange,
+			>,
+		) -> Vec<MessageNonce> {
+			source_queue.iter().flat_map(|(_, range)| range.clone()).collect()
+		}
+
+		strategy.remove_le_nonces_from_source_queue(1);
+		assert_eq!(source_queue_nonces(&strategy.source_queue), vec![2, 3, 4, 5, 6, 7, 8, 9],);
+
+		strategy.remove_le_nonces_from_source_queue(5);
+		assert_eq!(source_queue_nonces(&strategy.source_queue), vec![6, 7, 8, 9],);
+
+		strategy.remove_le_nonces_from_source_queue(9);
+		assert_eq!(source_queue_nonces(&strategy.source_queue), Vec::<MessageNonce>::new(),);
+
+		strategy.remove_le_nonces_from_source_queue(100);
+		assert_eq!(source_queue_nonces(&strategy.source_queue), Vec::<MessageNonce>::new(),);
+	}
+
+	#[async_std::test]
+	async fn previous_nonces_are_selected_if_reorg_happens_at_target_chain() {
+		let source_header_1 = header_id(1);
+		let target_header_1 = header_id(1);
+
+		// we start in perfect sync state - all headers are synced and finalized on both ends
+		let mut state = TestRaceStateImpl {
+			best_finalized_source_header_id_at_source: Some(source_header_1),
+			best_finalized_source_header_id_at_best_target: Some(source_header_1),
+			best_target_header_id: Some(target_header_1),
+			best_finalized_target_header_id: Some(target_header_1),
+			nonces_to_submit: None,
+			nonces_to_submit_batch: None,
+			nonces_submitted: None,
+		};
+
+		// in this state we have 1 available nonce for delivery
+		let mut strategy = BasicStrategy::<TestMessageLane> {
+			source_queue: vec![(header_id(1), 1..=1)].into_iter().collect(),
+			best_target_nonce: Some(0),
+			_phantom: PhantomData,
+		};
+		assert_eq!(strategy.select_nonces_to_deliver(state.clone()).await, Some((1..=1, ())),);
+
+		// let's say we have submitted 1..=1
+		state.nonces_submitted = Some(1..=1);
+
+		// then new nonce 2 appear at the source block 2
+		let source_header_2 = header_id(2);
+		state.best_finalized_source_header_id_at_source = Some(source_header_2);
+		strategy.source_nonces_updated(
+			source_header_2,
+			SourceClientNonces { new_nonces: 2..=2, confirmed_nonce: None },
+		);
+		// and nonce 1 appear at the best block of the target node (best finalized still has 0
+		// nonces)
+		let target_header_2 = header_id(2);
+		state.best_target_header_id = Some(target_header_2);
+		strategy.best_target_nonces_updated(
+			TargetClientNonces { latest_nonce: 1, nonces_data: () },
+			&mut state,
+		);
+
+		// then best target header is retracted
+		strategy.best_target_nonces_updated(
+			TargetClientNonces { latest_nonce: 0, nonces_data: () },
+			&mut state,
+		);
+
+		// ... and some fork with zero delivered nonces is finalized
+		let target_header_2_fork = header_id(2_1);
+		state.best_finalized_source_header_id_at_source = Some(source_header_2);
+		state.best_finalized_source_header_id_at_best_target = Some(source_header_2);
+		state.best_target_header_id = Some(target_header_2_fork);
+		state.best_finalized_target_header_id = Some(target_header_2_fork);
+		strategy.finalized_target_nonces_updated(
+			TargetClientNonces { latest_nonce: 0, nonces_data: () },
+			&mut state,
+		);
+
+		// now we have to select nonce 1 for delivery again
+		assert_eq!(strategy.select_nonces_to_deliver(state.clone()).await, Some((1..=2, ())),);
+	}
+}

pezbridges/relays/messages/src/metrics.rs

@@ -0,0 +1,177 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Metrics for message lane relay loop.
+
+use crate::{
+	message_lane::MessageLane,
+	message_lane_loop::{SourceClientState, TargetClientState},
+};
+
+use bp_messages::{HashedLaneId, LegacyLaneId, MessageNonce};
+use pez_finality_relay::SyncLoopMetrics;
+use relay_utils::metrics::{
+	metric_name, register, GaugeVec, Metric, Opts, PrometheusError, Registry, U64,
+};
+
+/// Message lane relay metrics.
+///
+/// Cloning only clones references.
+#[derive(Clone)]
+pub struct MessageLaneLoopMetrics {
+	/// Best finalized block numbers - "source", "source_at_target", "target_at_source".
+	source_to_target_finality_metrics: SyncLoopMetrics,
+	/// Best finalized block numbers - "source", "target", "source_at_target", "target_at_source".
+	target_to_source_finality_metrics: SyncLoopMetrics,
+	/// Lane state nonces: "source_latest_generated", "source_latest_confirmed",
+	/// "target_latest_received", "target_latest_confirmed".
+	lane_state_nonces: GaugeVec<U64>,
+}
+
+impl MessageLaneLoopMetrics {
+	/// Create and register messages loop metrics.
+	pub fn new(prefix: Option<&str>) -> Result<Self, PrometheusError> {
+		Ok(MessageLaneLoopMetrics {
+			source_to_target_finality_metrics: SyncLoopMetrics::new(
+				prefix,
+				"source",
+				"source_at_target",
+			)?,
+			target_to_source_finality_metrics: SyncLoopMetrics::new(
+				prefix,
+				"target",
+				"target_at_source",
+			)?,
+			lane_state_nonces: GaugeVec::new(
+				Opts::new(metric_name(prefix, "lane_state_nonces"), "Nonces of the lane state"),
+				&["type"],
+			)?,
+		})
+	}
+
+	/// Update source client state metrics.
+	pub fn update_source_state<P: MessageLane>(&self, source_client_state: SourceClientState<P>) {
+		self.source_to_target_finality_metrics
+			.update_best_block_at_source(source_client_state.best_self.0);
+		if let Some(best_finalized_peer_at_best_self) =
+			source_client_state.best_finalized_peer_at_best_self
+		{
+			self.target_to_source_finality_metrics
+				.update_best_block_at_target(best_finalized_peer_at_best_self.0);
+			if let Some(actual_best_finalized_peer_at_best_self) =
+				source_client_state.actual_best_finalized_peer_at_best_self
+			{
+				self.target_to_source_finality_metrics.update_using_same_fork(
+					best_finalized_peer_at_best_self.1 == actual_best_finalized_peer_at_best_self.1,
+				);
+			}
+		}
+	}
+
+	/// Update target client state metrics.
+	pub fn update_target_state<P: MessageLane>(&self, target_client_state: TargetClientState<P>) {
+		self.target_to_source_finality_metrics
+			.update_best_block_at_source(target_client_state.best_self.0);
+		if let Some(best_finalized_peer_at_best_self) =
+			target_client_state.best_finalized_peer_at_best_self
+		{
+			self.source_to_target_finality_metrics
+				.update_best_block_at_target(best_finalized_peer_at_best_self.0);
+			if let Some(actual_best_finalized_peer_at_best_self) =
+				target_client_state.actual_best_finalized_peer_at_best_self
+			{
+				self.source_to_target_finality_metrics.update_using_same_fork(
+					best_finalized_peer_at_best_self.1 == actual_best_finalized_peer_at_best_self.1,
+				);
+			}
+		}
+	}
+
+	/// Update latest generated nonce at source.
+	pub fn update_source_latest_generated_nonce(
+		&self,
+		source_latest_generated_nonce: MessageNonce,
+	) {
+		self.lane_state_nonces
+			.with_label_values(&["source_latest_generated"])
+			.set(source_latest_generated_nonce);
+	}
+
+	/// Update the latest confirmed nonce at source.
+	pub fn update_source_latest_confirmed_nonce(
+		&self,
+		source_latest_confirmed_nonce: MessageNonce,
+	) {
+		self.lane_state_nonces
+			.with_label_values(&["source_latest_confirmed"])
+			.set(source_latest_confirmed_nonce);
+	}
+
+	/// Update the latest received nonce at target.
+	pub fn update_target_latest_received_nonce(&self, target_latest_generated_nonce: MessageNonce) {
+		self.lane_state_nonces
+			.with_label_values(&["target_latest_received"])
+			.set(target_latest_generated_nonce);
+	}
+
+	/// Update the latest confirmed nonce at target.
+	pub fn update_target_latest_confirmed_nonce(
+		&self,
+		target_latest_confirmed_nonce: MessageNonce,
+	) {
+		self.lane_state_nonces
+			.with_label_values(&["target_latest_confirmed"])
+			.set(target_latest_confirmed_nonce);
+	}
+}
+
+impl Metric for MessageLaneLoopMetrics {
+	fn register(&self, registry: &Registry) -> Result<(), PrometheusError> {
+		self.source_to_target_finality_metrics.register(registry)?;
+		self.target_to_source_finality_metrics.register(registry)?;
+		register(self.lane_state_nonces.clone(), registry)?;
+		Ok(())
+	}
+}
+
+/// Provides a label for metrics.
+pub trait Labeled {
+	/// Returns a label.
+	fn label(&self) -> String;
+}
+
+/// `Labeled` implementation for `LegacyLaneId`.
+impl Labeled for LegacyLaneId {
+	fn label(&self) -> String {
+		hex::encode(self.0)
+	}
+}
+
+/// `Labeled` implementation for `HashedLaneId`.
+impl Labeled for HashedLaneId {
+	fn label(&self) -> String {
+		format!("{:?}", self.inner())
+	}
+}
+
+#[test]
+fn lane_to_label_works() {
+	assert_eq!(
+		"0x0101010101010101010101010101010101010101010101010101010101010101",
+		HashedLaneId::from_inner(pezsp_core::H256::from([1u8; 32])).label(),
+	);
+	assert_eq!("00000001", LegacyLaneId([0, 0, 0, 1]).label());
+}