fix: Complete snowbridge pezpallet rebrand and critical bug fixes

- snowbridge-pezpallet-* → pezsnowbridge-pezpallet-* (201 refs)
- pallet/ directories → pezpallet/ (4 locations)
- Fixed pezpallet.rs self-include recursion bug
- Fixed sc-chain-spec hardcoded crate name in derive macro
- Reverted .pezpallet_by_name() to .pallet_by_name() (subxt API)
- Added BizinikiwiConfig type alias for zombienet tests
- Deleted obsolete session state files

Verified: pezsnowbridge-pezpallet-*, pezpallet-staking,
pezpallet-staking-async, pezframe-benchmarking-cli all pass cargo check

pezbridges/relays/finality/src/base.rs

@@ -0,0 +1,47 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use async_trait::async_trait;
+use bp_header_pez_chain::FinalityProof;
+use futures::Stream;
+use relay_utils::relay_loop::Client as RelayClient;
+use std::fmt::Debug;
+
+/// Base finality pipeline.
+pub trait FinalityPipeline: 'static + Clone + Debug + Send + Sync {
+	/// Name of the finality proofs source.
+	const SOURCE_NAME: &'static str;
+	/// Name of the finality proofs target.
+	const TARGET_NAME: &'static str;
+
+	/// Synced headers are identified by this hash.
+	type Hash: Eq + Clone + Copy + Send + Sync + Debug;
+	/// Synced headers are identified by this number.
+	type Number: relay_utils::BlockNumberBase;
+	/// Finality proof type.
+	type FinalityProof: FinalityProof<Self::Hash, Self::Number>;
+}
+
+/// Source client used in finality related loops.
+#[async_trait]
+pub trait SourceClientBase<P: FinalityPipeline>: RelayClient {
+	/// Stream of new finality proofs. The stream is allowed to miss proofs for some
+	/// headers, even if those headers are mandatory.
+	type FinalityProofsStream: Stream<Item = P::FinalityProof> + Send + Unpin;
+
+	/// Subscribe to new finality proofs.
+	async fn finality_proofs(&self) -> Result<Self::FinalityProofsStream, Self::Error>;
+}

pezbridges/relays/finality/src/finality_loop.rs

@@ -0,0 +1,797 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The loop basically reads all missing headers and their finality proofs from the source client.
+//! The proof for the best possible header is then submitted to the target node. The only exception
+//! is the mandatory headers, which we always submit to the target node. For such headers, we
+//! assume that the persistent proof either exists, or will eventually become available.
+
+use crate::{sync_loop_metrics::SyncLoopMetrics, Error, FinalitySyncPipeline, SourceHeader};
+
+use crate::{
+	base::SourceClientBase,
+	finality_proofs::{FinalityProofsBuf, FinalityProofsStream},
+	headers::{JustifiedHeader, JustifiedHeaderSelector},
+};
+use async_trait::async_trait;
+use backoff::{backoff::Backoff, ExponentialBackoff};
+use futures::{future::Fuse, select, Future, FutureExt};
+use num_traits::{Saturating, Zero};
+use relay_utils::{
+	metrics::MetricsParams, relay_loop::Client as RelayClient, retry_backoff, FailedClient,
+	HeaderId, MaybeConnectionError, TrackedTransactionStatus, TransactionTracker,
+};
+use std::{
+	fmt::Debug,
+	time::{Duration, Instant},
+};
+
+/// Type of headers that we relay.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum HeadersToRelay {
+	/// Relay all headers.
+	All,
+	/// Relay only mandatory headers.
+	Mandatory,
+	/// Relay only free (including mandatory) headers.
+	Free,
+}
+
+/// Finality proof synchronization loop parameters.
+#[derive(Debug, Clone)]
+pub struct FinalitySyncParams {
+	/// Interval at which we check updates on both clients. Normally should be larger than
+	/// `min(source_block_time, target_block_time)`.
+	///
+	/// This parameter may be used to limit transactions rate. Increase the value && you'll get
+	/// infrequent updates => sparse headers => potential slow down of bridge applications, but
+	/// pezpallet storage won't be super large. Decrease the value to near `source_block_time` and
+	/// you'll get transaction for (almost) every block of the source chain => all source headers
+	/// will be known to the target chain => bridge applications will run faster, but pezpallet
+	/// storage may explode (but if pruning is there, then it's fine).
+	pub tick: Duration,
+	/// Number of finality proofs to keep in internal buffer between loop iterations.
+	///
+	/// While in "major syncing" state, we still read finality proofs from the stream. They're
+	/// stored in the internal buffer between loop iterations. When we're close to the tip of the
+	/// chain, we may meet finality delays if headers are not finalized frequently. So instead of
+	/// waiting for next finality proof to appear in the stream, we may use existing proof from
+	/// that buffer.
+	pub recent_finality_proofs_limit: usize,
+	/// Timeout before we treat our transactions as lost and restart the whole sync process.
+	pub stall_timeout: Duration,
+	/// If true, only mandatory headers are relayed.
+	pub headers_to_relay: HeadersToRelay,
+}
+
+/// Source client used in finality synchronization loop.
+#[async_trait]
+pub trait SourceClient<P: FinalitySyncPipeline>: SourceClientBase<P> {
+	/// Get best finalized block number.
+	async fn best_finalized_block_number(&self) -> Result<P::Number, Self::Error>;
+
+	/// Get canonical header and its finality proof by number.
+	async fn header_and_finality_proof(
+		&self,
+		number: P::Number,
+	) -> Result<(P::Header, Option<P::FinalityProof>), Self::Error>;
+}
+
+/// Target client used in finality synchronization loop.
+#[async_trait]
+pub trait TargetClient<P: FinalitySyncPipeline>: RelayClient {
+	/// Transaction tracker to track submitted transactions.
+	type TransactionTracker: TransactionTracker;
+
+	/// Get best finalized source block number.
+	async fn best_finalized_source_block_id(
+		&self,
+	) -> Result<HeaderId<P::Hash, P::Number>, Self::Error>;
+
+	/// Get free source headers submission interval, if it is configured in the
+	/// target runtime.
+	async fn free_source_headers_interval(&self) -> Result<Option<P::Number>, Self::Error>;
+
+	/// Submit header finality proof.
+	async fn submit_finality_proof(
+		&self,
+		header: P::Header,
+		proof: P::FinalityProof,
+		is_free_execution_expected: bool,
+	) -> Result<Self::TransactionTracker, Self::Error>;
+}
+
+/// Return prefix that will be used by default to expose Prometheus metrics of the finality proofs
+/// sync loop.
+pub fn metrics_prefix<P: FinalitySyncPipeline>() -> String {
+	format!("{}_to_{}_Sync", P::SOURCE_NAME, P::TARGET_NAME)
+}
+
+/// Finality sync information.
+pub struct SyncInfo<P: FinalitySyncPipeline> {
+	/// Best finalized header at the source client.
+	pub best_number_at_source: P::Number,
+	/// Best source header, known to the target client.
+	pub best_number_at_target: P::Number,
+	/// Whether the target client follows the same fork as the source client do.
+	pub is_using_same_fork: bool,
+}
+
+impl<P: FinalitySyncPipeline> SyncInfo<P> {
+	/// Checks if both clients are on the same fork.
+	async fn is_on_same_fork<SC: SourceClient<P>>(
+		source_client: &SC,
+		id_at_target: &HeaderId<P::Hash, P::Number>,
+	) -> Result<bool, SC::Error> {
+		let header_at_source = source_client.header_and_finality_proof(id_at_target.0).await?.0;
+		let header_hash_at_source = header_at_source.hash();
+		Ok(if id_at_target.1 == header_hash_at_source {
+			true
+		} else {
+			tracing::error!(
+				target: "bridge",
+				source=%P::SOURCE_NAME,
+				target=%P::TARGET_NAME,
+				height=?id_at_target.0,
+				at_source=?header_hash_at_source,
+				at_target=?id_at_target.1,
+				"Source node and pezpallet at target node have different headers at the same height"
+			);
+
+			false
+		})
+	}
+
+	async fn new<SC: SourceClient<P>, TC: TargetClient<P>>(
+		source_client: &SC,
+		target_client: &TC,
+	) -> Result<Self, Error<P, SC::Error, TC::Error>> {
+		let best_number_at_source =
+			source_client.best_finalized_block_number().await.map_err(Error::Source)?;
+		let best_id_at_target =
+			target_client.best_finalized_source_block_id().await.map_err(Error::Target)?;
+		let best_number_at_target = best_id_at_target.0;
+
+		let is_using_same_fork = Self::is_on_same_fork(source_client, &best_id_at_target)
+			.await
+			.map_err(Error::Source)?;
+
+		Ok(Self { best_number_at_source, best_number_at_target, is_using_same_fork })
+	}
+
+	fn update_metrics(&self, metrics_sync: &Option<SyncLoopMetrics>) {
+		if let Some(metrics_sync) = metrics_sync {
+			metrics_sync.update_best_block_at_source(self.best_number_at_source);
+			metrics_sync.update_best_block_at_target(self.best_number_at_target);
+			metrics_sync.update_using_same_fork(self.is_using_same_fork);
+		}
+	}
+
+	pub fn num_headers(&self) -> P::Number {
+		self.best_number_at_source.saturating_sub(self.best_number_at_target)
+	}
+}
+
+/// Information about transaction that we have submitted.
+#[derive(Debug, Clone)]
+pub struct Transaction<Tracker, Number> {
+	/// Submitted transaction tracker.
+	tracker: Tracker,
+	/// The number of the header we have submitted.
+	header_number: Number,
+}
+
+impl<Tracker: TransactionTracker, Number: Debug + PartialOrd> Transaction<Tracker, Number> {
+	pub async fn submit<
+		P: FinalitySyncPipeline<Number = Number>,
+		TC: TargetClient<P, TransactionTracker = Tracker>,
+	>(
+		target_client: &TC,
+		header: P::Header,
+		justification: P::FinalityProof,
+		is_free_execution_expected: bool,
+	) -> Result<Self, TC::Error> {
+		let header_number = header.number();
+		tracing::debug!(
+			target: "bridge",
+			source=%P::SOURCE_NAME,
+			target=%P::TARGET_NAME,
+			header=?header_number,
+			"Going to submit finality proof of header"
+		);
+
+		let tracker = target_client
+			.submit_finality_proof(header, justification, is_free_execution_expected)
+			.await?;
+		Ok(Transaction { tracker, header_number })
+	}
+
+	async fn track<
+		P: FinalitySyncPipeline<Number = Number>,
+		SC: SourceClient<P>,
+		TC: TargetClient<P>,
+	>(
+		self,
+		target_client: TC,
+	) -> Result<(), Error<P, SC::Error, TC::Error>> {
+		match self.tracker.wait().await {
+			TrackedTransactionStatus::Finalized(_) => {
+				// The transaction has been finalized, but it may have been finalized in the
+				// "failed" state. So let's check if the block number was actually updated.
+				target_client
+					.best_finalized_source_block_id()
+					.await
+					.map_err(Error::Target)
+					.and_then(|best_id_at_target| {
+						if self.header_number > best_id_at_target.0 {
+							return Err(Error::ProofSubmissionTxFailed {
+								submitted_number: self.header_number,
+								best_number_at_target: best_id_at_target.0,
+							});
+						}
+						Ok(())
+					})
+			},
+			TrackedTransactionStatus::Lost => Err(Error::ProofSubmissionTxLost),
+		}
+	}
+}
+
+/// Finality synchronization loop state.
+struct FinalityLoop<P: FinalitySyncPipeline, SC: SourceClient<P>, TC: TargetClient<P>> {
+	source_client: SC,
+	target_client: TC,
+
+	sync_params: FinalitySyncParams,
+	metrics_sync: Option<SyncLoopMetrics>,
+
+	progress: (Instant, Option<P::Number>),
+	retry_backoff: ExponentialBackoff,
+	finality_proofs_stream: FinalityProofsStream<P, SC>,
+	finality_proofs_buf: FinalityProofsBuf<P>,
+	best_submitted_number: Option<P::Number>,
+}
+
+impl<P: FinalitySyncPipeline, SC: SourceClient<P>, TC: TargetClient<P>> FinalityLoop<P, SC, TC> {
+	pub fn new(
+		source_client: SC,
+		target_client: TC,
+		sync_params: FinalitySyncParams,
+		metrics_sync: Option<SyncLoopMetrics>,
+	) -> Self {
+		Self {
+			source_client,
+			target_client,
+			sync_params,
+			metrics_sync,
+			progress: (Instant::now(), None),
+			retry_backoff: retry_backoff(),
+			finality_proofs_stream: FinalityProofsStream::new(),
+			finality_proofs_buf: FinalityProofsBuf::new(vec![]),
+			best_submitted_number: None,
+		}
+	}
+
+	fn update_progress(&mut self, info: &SyncInfo<P>) {
+		let (prev_time, prev_best_number_at_target) = self.progress;
+		let now = Instant::now();
+
+		let needs_update = now - prev_time > Duration::from_secs(10) ||
+			prev_best_number_at_target
+				.map(|prev_best_number_at_target| {
+					info.best_number_at_target.saturating_sub(prev_best_number_at_target) >
+						10.into()
+				})
+				.unwrap_or(true);
+
+		if !needs_update {
+			return;
+		}
+
+		tracing::info!(
+			target: "bridge",
+			best_number_at_target=?info.best_number_at_target,
+			best_number_at_source=?info.best_number_at_source,
+			"Synced headers"
+		);
+
+		self.progress = (now, Some(info.best_number_at_target))
+	}
+
+	pub async fn select_header_to_submit(
+		&mut self,
+		info: &SyncInfo<P>,
+		free_headers_interval: Option<P::Number>,
+	) -> Result<Option<JustifiedHeader<P>>, Error<P, SC::Error, TC::Error>> {
+		// to see that the loop is progressing
+		tracing::trace!(
+			target: "bridge",
+			best_number_at_target=%info.best_number_at_target,
+			best_number_at_source=%info.best_number_at_source,
+			"Considering range of headers"
+		);
+
+		// read missing headers
+		let selector = JustifiedHeaderSelector::new::<SC, TC>(
+			&self.source_client,
+			info,
+			self.sync_params.headers_to_relay,
+			free_headers_interval,
+		)
+		.await?;
+		// if we see that the header schedules GRANDPA change, we need to submit it
+		if self.sync_params.headers_to_relay == HeadersToRelay::Mandatory {
+			return Ok(selector.select_mandatory());
+		}
+
+		// all headers that are missing from the target client are non-mandatory
+		// => even if we have already selected some header and its persistent finality proof,
+		// we may try to select better header by reading non-persistent proofs from the stream
+		self.finality_proofs_buf.fill(&mut self.finality_proofs_stream);
+		let maybe_justified_header = selector.select(
+			info,
+			self.sync_params.headers_to_relay,
+			free_headers_interval,
+			&self.finality_proofs_buf,
+		);
+
+		// remove obsolete 'recent' finality proofs + keep its size under certain limit
+		let oldest_finality_proof_to_keep = maybe_justified_header
+			.as_ref()
+			.map(|justified_header| justified_header.number())
+			.unwrap_or(info.best_number_at_target);
+		self.finality_proofs_buf.prune(
+			oldest_finality_proof_to_keep,
+			Some(self.sync_params.recent_finality_proofs_limit),
+		);
+
+		Ok(maybe_justified_header)
+	}
+
+	pub async fn run_iteration(
+		&mut self,
+		free_headers_interval: Option<P::Number>,
+	) -> Result<
+		Option<Transaction<TC::TransactionTracker, P::Number>>,
+		Error<P, SC::Error, TC::Error>,
+	> {
+		// read best source headers ids from source and target nodes
+		let info = SyncInfo::new(&self.source_client, &self.target_client).await?;
+		info.update_metrics(&self.metrics_sync);
+		self.update_progress(&info);
+
+		// if we have already submitted header, then we just need to wait for it
+		// if we're waiting too much, then we believe our transaction has been lost and restart sync
+		if Some(info.best_number_at_target) < self.best_submitted_number {
+			return Ok(None);
+		}
+
+		// submit new header if we have something new
+		match self.select_header_to_submit(&info, free_headers_interval).await? {
+			Some(header) => {
+				let transaction = Transaction::submit(
+					&self.target_client,
+					header.header,
+					header.proof,
+					self.sync_params.headers_to_relay == HeadersToRelay::Free,
+				)
+				.await
+				.map_err(Error::Target)?;
+				self.best_submitted_number = Some(transaction.header_number);
+				Ok(Some(transaction))
+			},
+			None => Ok(None),
+		}
+	}
+
+	async fn ensure_finality_proofs_stream(&mut self) -> Result<(), FailedClient> {
+		if let Err(e) = self.finality_proofs_stream.ensure_stream(&self.source_client).await {
+			if e.is_connection_error() {
+				return Err(FailedClient::Source);
+			}
+		}
+
+		Ok(())
+	}
+
+	/// Run finality relay loop until connection to one of nodes is lost.
+	async fn run_until_connection_lost(
+		&mut self,
+		exit_signal: impl Future<Output = ()>,
+	) -> Result<(), FailedClient> {
+		self.ensure_finality_proofs_stream().await?;
+		let proof_submission_tx_tracker = Fuse::terminated();
+		let exit_signal = exit_signal.fuse();
+		futures::pin_mut!(exit_signal, proof_submission_tx_tracker);
+
+		let free_headers_interval = free_headers_interval(&self.target_client).await?;
+
+		loop {
+			// run loop iteration
+			let next_tick = match self.run_iteration(free_headers_interval).await {
+				Ok(Some(tx)) => {
+					proof_submission_tx_tracker
+						.set(tx.track::<P, SC, _>(self.target_client.clone()).fuse());
+					self.retry_backoff.reset();
+					self.sync_params.tick
+				},
+				Ok(None) => {
+					self.retry_backoff.reset();
+					self.sync_params.tick
+				},
+				Err(error) => {
+					tracing::error!(target: "bridge", ?error, "Finality sync loop iteration has failed");
+					error.fail_if_connection_error()?;
+					self.retry_backoff
+						.next_backoff()
+						.unwrap_or(relay_utils::relay_loop::RECONNECT_DELAY)
+				},
+			};
+			self.ensure_finality_proofs_stream().await?;
+
+			// wait till exit signal, or new source block
+			select! {
+				proof_submission_result = proof_submission_tx_tracker => {
+					if let Err(e) = proof_submission_result {
+						tracing::error!(
+							target: "bridge",
+							error=?e,
+							target=%P::TARGET_NAME,
+							"Finality sync proof submission tx has failed."
+						);
+						self.best_submitted_number = None;
+						e.fail_if_connection_error()?;
+					}
+				},
+				_ = async_std::task::sleep(next_tick).fuse() => {},
+				_ = exit_signal => return Ok(()),
+			}
+		}
+	}
+
+	pub async fn run(
+		source_client: SC,
+		target_client: TC,
+		sync_params: FinalitySyncParams,
+		metrics_sync: Option<SyncLoopMetrics>,
+		exit_signal: impl Future<Output = ()>,
+	) -> Result<(), FailedClient> {
+		let mut finality_loop = Self::new(source_client, target_client, sync_params, metrics_sync);
+		finality_loop.run_until_connection_lost(exit_signal).await
+	}
+}
+
+async fn free_headers_interval<P: FinalitySyncPipeline>(
+	target_client: &impl TargetClient<P>,
+) -> Result<Option<P::Number>, FailedClient> {
+	match target_client.free_source_headers_interval().await {
+		Ok(Some(free_headers_interval)) if !free_headers_interval.is_zero() => {
+			tracing::trace!(
+				target: "bridge",
+				source=%P::SOURCE_NAME,
+				target=%P::TARGET_NAME,
+				?free_headers_interval,
+				"Free headers interval for headers"
+			);
+			Ok(Some(free_headers_interval))
+		},
+		Ok(Some(_free_headers_interval)) => {
+			tracing::trace!(
+				target: "bridge",
+				source=%P::SOURCE_NAME,
+				target=%P::TARGET_NAME,
+				"Free headers interval for headers is zero. Not submitting any free headers"
+			);
+			Ok(None)
+		},
+		Ok(None) => {
+			tracing::trace!(
+				target: "bridge",
+				source=%P::SOURCE_NAME,
+				target=%P::TARGET_NAME,
+				"Free headers interval for headers is None. Not submitting any free headers"
+			);
+
+			Ok(None)
+		},
+		Err(e) => {
+			tracing::error!(
+				target: "bridge",
+				error=?e,
+				source=%P::SOURCE_NAME,
+				target=%P::TARGET_NAME,
+				"Failed to read free headers interval for headers"
+			);
+			Err(FailedClient::Target)
+		},
+	}
+}
+
+/// Run finality proofs synchronization loop.
+pub async fn run<P: FinalitySyncPipeline>(
+	source_client: impl SourceClient<P>,
+	target_client: impl TargetClient<P>,
+	sync_params: FinalitySyncParams,
+	metrics_params: MetricsParams,
+	exit_signal: impl Future<Output = ()> + 'static + Send,
+) -> Result<(), relay_utils::Error> {
+	let exit_signal = exit_signal.shared();
+	relay_utils::relay_loop(source_client, target_client)
+		.with_metrics(metrics_params)
+		.loop_metric(SyncLoopMetrics::new(
+			Some(&metrics_prefix::<P>()),
+			"source",
+			"source_at_target",
+		)?)?
+		.expose()
+		.await?
+		.run(metrics_prefix::<P>(), move |source_client, target_client, metrics| {
+			FinalityLoop::run(
+				source_client,
+				target_client,
+				sync_params.clone(),
+				metrics,
+				exit_signal.clone(),
+			)
+		})
+		.await
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	use crate::mock::*;
+	use futures::{FutureExt, StreamExt};
+	use parking_lot::Mutex;
+	use relay_utils::{FailedClient, HeaderId, TrackedTransactionStatus};
+	use std::{collections::HashMap, sync::Arc};
+
+	fn prepare_test_clients(
+		exit_sender: futures::channel::mpsc::UnboundedSender<()>,
+		state_function: impl Fn(&mut ClientsData) -> bool + Send + Sync + 'static,
+		source_headers: HashMap<TestNumber, (TestSourceHeader, Option<TestFinalityProof>)>,
+	) -> (TestSourceClient, TestTargetClient) {
+		let internal_state_function: Arc<dyn Fn(&mut ClientsData) + Send + Sync> =
+			Arc::new(move |data| {
+				if state_function(data) {
+					exit_sender.unbounded_send(()).unwrap();
+				}
+			});
+		let clients_data = Arc::new(Mutex::new(ClientsData {
+			source_best_block_number: 10,
+			source_headers,
+			source_proofs: vec![TestFinalityProof(12), TestFinalityProof(14)],
+
+			target_best_block_id: HeaderId(5, 5),
+			target_headers: vec![],
+			target_transaction_tracker: TestTransactionTracker(
+				TrackedTransactionStatus::Finalized(Default::default()),
+			),
+		}));
+		(
+			TestSourceClient {
+				on_method_call: internal_state_function.clone(),
+				data: clients_data.clone(),
+			},
+			TestTargetClient { on_method_call: internal_state_function, data: clients_data },
+		)
+	}
+
+	fn test_sync_params() -> FinalitySyncParams {
+		FinalitySyncParams {
+			tick: Duration::from_secs(0),
+			recent_finality_proofs_limit: 1024,
+			stall_timeout: Duration::from_secs(1),
+			headers_to_relay: HeadersToRelay::All,
+		}
+	}
+
+	fn run_sync_loop(
+		state_function: impl Fn(&mut ClientsData) -> bool + Send + Sync + 'static,
+	) -> (ClientsData, Result<(), FailedClient>) {
+		let (exit_sender, exit_receiver) = futures::channel::mpsc::unbounded();
+		let (source_client, target_client) = prepare_test_clients(
+			exit_sender,
+			state_function,
+			vec![
+				(5, (TestSourceHeader(false, 5, 5), None)),
+				(6, (TestSourceHeader(false, 6, 6), None)),
+				(7, (TestSourceHeader(false, 7, 7), Some(TestFinalityProof(7)))),
+				(8, (TestSourceHeader(true, 8, 8), Some(TestFinalityProof(8)))),
+				(9, (TestSourceHeader(false, 9, 9), Some(TestFinalityProof(9)))),
+				(10, (TestSourceHeader(false, 10, 10), None)),
+			]
+			.into_iter()
+			.collect(),
+		);
+		let sync_params = test_sync_params();
+
+		let clients_data = source_client.data.clone();
+		let result = async_std::task::block_on(FinalityLoop::run(
+			source_client,
+			target_client,
+			sync_params,
+			None,
+			exit_receiver.into_future().map(|(_, _)| ()),
+		));
+
+		let clients_data = clients_data.lock().clone();
+		(clients_data, result)
+	}
+
+	#[test]
+	fn finality_sync_loop_works() {
+		let (client_data, result) = run_sync_loop(|data| {
+			// header#7 has persistent finality proof, but it isn't mandatory => it isn't submitted,
+			// because header#8 has persistent finality proof && it is mandatory => it is submitted
+			// header#9 has persistent finality proof, but it isn't mandatory => it is submitted,
+			// because   there are no more persistent finality proofs
+			//
+			// once this ^^^ is done, we generate more blocks && read proof for blocks 12 and 14
+			// from the stream
+			if data.target_best_block_id.0 == 9 {
+				data.source_best_block_number = 14;
+				data.source_headers.insert(11, (TestSourceHeader(false, 11, 11), None));
+				data.source_headers
+					.insert(12, (TestSourceHeader(false, 12, 12), Some(TestFinalityProof(12))));
+				data.source_headers.insert(13, (TestSourceHeader(false, 13, 13), None));
+				data.source_headers
+					.insert(14, (TestSourceHeader(false, 14, 14), Some(TestFinalityProof(14))));
+			}
+			// once this ^^^ is done, we generate more blocks && read persistent proof for block 16
+			if data.target_best_block_id.0 == 14 {
+				data.source_best_block_number = 17;
+				data.source_headers.insert(15, (TestSourceHeader(false, 15, 15), None));
+				data.source_headers
+					.insert(16, (TestSourceHeader(false, 16, 16), Some(TestFinalityProof(16))));
+				data.source_headers.insert(17, (TestSourceHeader(false, 17, 17), None));
+			}
+
+			data.target_best_block_id.0 == 16
+		});
+
+		assert_eq!(result, Ok(()));
+		assert_eq!(
+			client_data.target_headers,
+			vec![
+				// before adding 11..14: finality proof for mandatory header#8
+				(TestSourceHeader(true, 8, 8), TestFinalityProof(8)),
+				// before adding 11..14: persistent finality proof for non-mandatory header#9
+				(TestSourceHeader(false, 9, 9), TestFinalityProof(9)),
+				// after adding 11..14: ephemeral finality proof for non-mandatory header#14
+				(TestSourceHeader(false, 14, 14), TestFinalityProof(14)),
+				// after adding 15..17: persistent finality proof for non-mandatory header#16
+				(TestSourceHeader(false, 16, 16), TestFinalityProof(16)),
+			],
+		);
+	}
+
+	fn run_headers_to_relay_mode_test(
+		headers_to_relay: HeadersToRelay,
+		has_mandatory_headers: bool,
+	) -> Option<JustifiedHeader<TestFinalitySyncPipeline>> {
+		let (exit_sender, _) = futures::channel::mpsc::unbounded();
+		let (source_client, target_client) = prepare_test_clients(
+			exit_sender,
+			|_| false,
+			vec![
+				(6, (TestSourceHeader(false, 6, 6), Some(TestFinalityProof(6)))),
+				(7, (TestSourceHeader(false, 7, 7), Some(TestFinalityProof(7)))),
+				(8, (TestSourceHeader(has_mandatory_headers, 8, 8), Some(TestFinalityProof(8)))),
+				(9, (TestSourceHeader(false, 9, 9), Some(TestFinalityProof(9)))),
+				(10, (TestSourceHeader(false, 10, 10), Some(TestFinalityProof(10)))),
+			]
+			.into_iter()
+			.collect(),
+		);
+		async_std::task::block_on(async {
+			let mut finality_loop = FinalityLoop::new(
+				source_client,
+				target_client,
+				FinalitySyncParams {
+					tick: Duration::from_secs(0),
+					recent_finality_proofs_limit: 0,
+					stall_timeout: Duration::from_secs(0),
+					headers_to_relay,
+				},
+				None,
+			);
+			let info = SyncInfo {
+				best_number_at_source: 10,
+				best_number_at_target: 5,
+				is_using_same_fork: true,
+			};
+			finality_loop.select_header_to_submit(&info, Some(3)).await.unwrap()
+		})
+	}
+
+	#[test]
+	fn select_header_to_submit_may_select_non_mandatory_header() {
+		assert_eq!(run_headers_to_relay_mode_test(HeadersToRelay::Mandatory, false), None);
+		assert_eq!(
+			run_headers_to_relay_mode_test(HeadersToRelay::Free, false),
+			Some(JustifiedHeader {
+				header: TestSourceHeader(false, 10, 10),
+				proof: TestFinalityProof(10)
+			}),
+		);
+		assert_eq!(
+			run_headers_to_relay_mode_test(HeadersToRelay::All, false),
+			Some(JustifiedHeader {
+				header: TestSourceHeader(false, 10, 10),
+				proof: TestFinalityProof(10)
+			}),
+		);
+	}
+
+	#[test]
+	fn select_header_to_submit_may_select_mandatory_header() {
+		assert_eq!(
+			run_headers_to_relay_mode_test(HeadersToRelay::Mandatory, true),
+			Some(JustifiedHeader {
+				header: TestSourceHeader(true, 8, 8),
+				proof: TestFinalityProof(8)
+			}),
+		);
+		assert_eq!(
+			run_headers_to_relay_mode_test(HeadersToRelay::Free, true),
+			Some(JustifiedHeader {
+				header: TestSourceHeader(true, 8, 8),
+				proof: TestFinalityProof(8)
+			}),
+		);
+		assert_eq!(
+			run_headers_to_relay_mode_test(HeadersToRelay::All, true),
+			Some(JustifiedHeader {
+				header: TestSourceHeader(true, 8, 8),
+				proof: TestFinalityProof(8)
+			}),
+		);
+	}
+
+	#[test]
+	fn different_forks_at_source_and_at_target_are_detected() {
+		let (exit_sender, _exit_receiver) = futures::channel::mpsc::unbounded();
+		let (source_client, target_client) = prepare_test_clients(
+			exit_sender,
+			|_| false,
+			vec![
+				(5, (TestSourceHeader(false, 5, 42), None)),
+				(6, (TestSourceHeader(false, 6, 6), None)),
+				(7, (TestSourceHeader(false, 7, 7), None)),
+				(8, (TestSourceHeader(false, 8, 8), None)),
+				(9, (TestSourceHeader(false, 9, 9), None)),
+				(10, (TestSourceHeader(false, 10, 10), None)),
+			]
+			.into_iter()
+			.collect(),
+		);
+
+		let metrics_sync = SyncLoopMetrics::new(None, "source", "target").unwrap();
+		async_std::task::block_on(async {
+			let mut finality_loop = FinalityLoop::new(
+				source_client,
+				target_client,
+				test_sync_params(),
+				Some(metrics_sync.clone()),
+			);
+			finality_loop.run_iteration(None).await.unwrap()
+		});
+
+		assert!(!metrics_sync.is_using_same_fork());
+	}
+}

pezbridges/relays/finality/src/finality_proofs.rs

@@ -0,0 +1,221 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{base::SourceClientBase, FinalityPipeline};
+
+use bp_header_pez_chain::FinalityProof;
+use futures::{FutureExt, Stream, StreamExt};
+use std::pin::Pin;
+
+/// Source finality proofs stream that may be restarted.
+#[derive(Default)]
+pub struct FinalityProofsStream<P: FinalityPipeline, SC: SourceClientBase<P>> {
+	/// The underlying stream.
+	stream: Option<Pin<Box<SC::FinalityProofsStream>>>,
+}
+
+impl<P: FinalityPipeline, SC: SourceClientBase<P>> FinalityProofsStream<P, SC> {
+	pub fn new() -> Self {
+		Self { stream: None }
+	}
+
+	pub fn from_stream(stream: SC::FinalityProofsStream) -> Self {
+		Self { stream: Some(Box::pin(stream)) }
+	}
+
+	fn next(&mut self) -> Option<<SC::FinalityProofsStream as Stream>::Item> {
+		let stream = match &mut self.stream {
+			Some(stream) => stream,
+			None => return None,
+		};
+
+		match stream.next().now_or_never() {
+			Some(Some(finality_proof)) => Some(finality_proof),
+			Some(None) => {
+				self.stream = None;
+				None
+			},
+			None => None,
+		}
+	}
+
+	pub async fn ensure_stream(&mut self, source_client: &SC) -> Result<(), SC::Error> {
+		if self.stream.is_none() {
+			tracing::warn!(target: "bridge", source=%P::SOURCE_NAME, "Finality proofs stream is being started / restarted");
+
+			let stream = source_client.finality_proofs().await.map_err(|error| {
+				tracing::error!(
+					target: "bridge",
+					?error,
+					source=%P::SOURCE_NAME,
+					"Failed to subscribe to justifications"
+				);
+
+				error
+			})?;
+			self.stream = Some(Box::pin(stream));
+		}
+
+		Ok(())
+	}
+}
+
+/// Source finality proofs buffer.
+pub struct FinalityProofsBuf<P: FinalityPipeline> {
+	/// Proofs buffer. Ordered by target header number.
+	buf: Vec<P::FinalityProof>,
+}
+
+impl<P: FinalityPipeline> FinalityProofsBuf<P> {
+	pub fn new(buf: Vec<P::FinalityProof>) -> Self {
+		Self { buf }
+	}
+
+	pub fn buf(&self) -> &Vec<P::FinalityProof> {
+		&self.buf
+	}
+
+	pub fn fill<SC: SourceClientBase<P>>(&mut self, stream: &mut FinalityProofsStream<P, SC>) {
+		let mut proofs_count = 0;
+		let mut first_header_number = None;
+		let mut last_header_number = None;
+		while let Some(finality_proof) = stream.next() {
+			let target_header_number = finality_proof.target_header_number();
+			first_header_number.get_or_insert(target_header_number);
+			last_header_number = Some(target_header_number);
+			proofs_count += 1;
+
+			self.buf.push(finality_proof);
+		}
+
+		if proofs_count != 0 {
+			tracing::trace!(
+				target: "bridge",
+				source=%P::SOURCE_NAME,
+				%proofs_count,
+				?first_header_number,
+				?last_header_number,
+				"Read finality proofs from finality stream for headers in range",
+			);
+		}
+	}
+
+	/// Prune all finality proofs that target header numbers older than `first_to_keep`.
+	pub fn prune(&mut self, first_to_keep: P::Number, maybe_buf_limit: Option<usize>) {
+		let first_to_keep_idx = self
+			.buf
+			.binary_search_by_key(&first_to_keep, |hdr| hdr.target_header_number())
+			.map(|idx| idx + 1)
+			.unwrap_or_else(|idx| idx);
+		let buf_limit_idx = match maybe_buf_limit {
+			Some(buf_limit) => self.buf.len().saturating_sub(buf_limit),
+			None => 0,
+		};
+
+		self.buf = self.buf.split_off(std::cmp::max(first_to_keep_idx, buf_limit_idx));
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::mock::*;
+
+	#[test]
+	fn finality_proofs_buf_fill_works() {
+		// when stream is currently empty, nothing is changed
+		let mut finality_proofs_buf =
+			FinalityProofsBuf::<TestFinalitySyncPipeline> { buf: vec![TestFinalityProof(1)] };
+		let mut stream =
+			FinalityProofsStream::<TestFinalitySyncPipeline, TestSourceClient>::from_stream(
+				Box::pin(futures::stream::pending()),
+			);
+		finality_proofs_buf.fill(&mut stream);
+		assert_eq!(finality_proofs_buf.buf, vec![TestFinalityProof(1)]);
+		assert!(stream.stream.is_some());
+
+		// when stream has entry with target, it is added to the recent proofs container
+		let mut stream =
+			FinalityProofsStream::<TestFinalitySyncPipeline, TestSourceClient>::from_stream(
+				Box::pin(
+					futures::stream::iter(vec![TestFinalityProof(4)])
+						.chain(futures::stream::pending()),
+				),
+			);
+		finality_proofs_buf.fill(&mut stream);
+		assert_eq!(finality_proofs_buf.buf, vec![TestFinalityProof(1), TestFinalityProof(4)]);
+		assert!(stream.stream.is_some());
+
+		// when stream has ended, we'll need to restart it
+		let mut stream =
+			FinalityProofsStream::<TestFinalitySyncPipeline, TestSourceClient>::from_stream(
+				Box::pin(futures::stream::empty()),
+			);
+		finality_proofs_buf.fill(&mut stream);
+		assert_eq!(finality_proofs_buf.buf, vec![TestFinalityProof(1), TestFinalityProof(4)]);
+		assert!(stream.stream.is_none());
+	}
+
+	#[test]
+	fn finality_proofs_buf_prune_works() {
+		let original_finality_proofs_buf: Vec<
+			<TestFinalitySyncPipeline as FinalityPipeline>::FinalityProof,
+		> = vec![
+			TestFinalityProof(10),
+			TestFinalityProof(13),
+			TestFinalityProof(15),
+			TestFinalityProof(17),
+			TestFinalityProof(19),
+		]
+		.into_iter()
+		.collect();
+
+		// when there's proof for justified header in the vec
+		let mut finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline> {
+			buf: original_finality_proofs_buf.clone(),
+		};
+		finality_proofs_buf.prune(10, None);
+		assert_eq!(&original_finality_proofs_buf[1..], finality_proofs_buf.buf,);
+
+		// when there are no proof for justified header in the vec
+		let mut finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline> {
+			buf: original_finality_proofs_buf.clone(),
+		};
+		finality_proofs_buf.prune(11, None);
+		assert_eq!(&original_finality_proofs_buf[1..], finality_proofs_buf.buf,);
+
+		// when there are too many entries after initial prune && they also need to be pruned
+		let mut finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline> {
+			buf: original_finality_proofs_buf.clone(),
+		};
+		finality_proofs_buf.prune(10, Some(2));
+		assert_eq!(&original_finality_proofs_buf[3..], finality_proofs_buf.buf,);
+
+		// when last entry is pruned
+		let mut finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline> {
+			buf: original_finality_proofs_buf.clone(),
+		};
+		finality_proofs_buf.prune(19, Some(2));
+		assert_eq!(&original_finality_proofs_buf[5..], finality_proofs_buf.buf,);
+
+		// when post-last entry is pruned
+		let mut finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline> {
+			buf: original_finality_proofs_buf.clone(),
+		};
+		finality_proofs_buf.prune(20, Some(2));
+		assert_eq!(&original_finality_proofs_buf[5..], finality_proofs_buf.buf,);
+	}
+}

pezbridges/relays/finality/src/headers.rs

@@ -0,0 +1,361 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{
+	finality_loop::SyncInfo, finality_proofs::FinalityProofsBuf, Error, FinalitySyncPipeline,
+	HeadersToRelay, SourceClient, SourceHeader, TargetClient,
+};
+
+use bp_header_pez_chain::FinalityProof;
+use num_traits::Saturating;
+use std::cmp::Ordering;
+
+/// Unjustified headers container. Ordered by header number.
+pub type UnjustifiedHeaders<H> = Vec<H>;
+
+#[derive(Debug)]
+#[cfg_attr(test, derive(Clone, PartialEq))]
+pub struct JustifiedHeader<P: FinalitySyncPipeline> {
+	pub header: P::Header,
+	pub proof: P::FinalityProof,
+}
+
+impl<P: FinalitySyncPipeline> JustifiedHeader<P> {
+	pub fn number(&self) -> P::Number {
+		self.header.number()
+	}
+}
+
+/// Finality proof that has been selected by the `read_missing_headers` function.
+pub enum JustifiedHeaderSelector<P: FinalitySyncPipeline> {
+	/// Mandatory header and its proof has been selected. We shall submit proof for this header.
+	Mandatory(JustifiedHeader<P>),
+	/// Regular header and its proof has been selected. We may submit this proof, or proof for
+	/// some better header.
+	Regular(UnjustifiedHeaders<P::Header>, JustifiedHeader<P>),
+	/// We haven't found any missing header with persistent proof at the target client.
+	None(UnjustifiedHeaders<P::Header>),
+}
+
+impl<P: FinalitySyncPipeline> JustifiedHeaderSelector<P> {
+	/// Selects last header with persistent justification, missing from the target and matching
+	/// the `headers_to_relay` criteria.
+	pub(crate) async fn new<SC: SourceClient<P>, TC: TargetClient<P>>(
+		source_client: &SC,
+		info: &SyncInfo<P>,
+		headers_to_relay: HeadersToRelay,
+		free_headers_interval: Option<P::Number>,
+	) -> Result<Self, Error<P, SC::Error, TC::Error>> {
+		let mut unjustified_headers = Vec::new();
+		let mut maybe_justified_header = None;
+
+		let mut header_number = info.best_number_at_target + 1.into();
+		while header_number <= info.best_number_at_source {
+			let (header, maybe_proof) = source_client
+				.header_and_finality_proof(header_number)
+				.await
+				.map_err(Error::Source)?;
+
+			match (header.is_mandatory(), maybe_proof) {
+				(true, Some(proof)) => {
+					tracing::trace!(target: "bridge", ?header_number, "Header is mandatory");
+					return Ok(Self::Mandatory(JustifiedHeader { header, proof }));
+				},
+				(true, None) => return Err(Error::MissingMandatoryFinalityProof(header.number())),
+				(false, Some(proof))
+					if need_to_relay::<P>(
+						info,
+						headers_to_relay,
+						free_headers_interval,
+						&header,
+					) =>
+				{
+					tracing::trace!(target: "bridge", ?header_number, "Header has persistent finality proof");
+					unjustified_headers.clear();
+					maybe_justified_header = Some(JustifiedHeader { header, proof });
+				},
+				_ => {
+					unjustified_headers.push(header);
+				},
+			}
+
+			header_number = header_number + 1.into();
+		}
+
+		tracing::trace!(
+			target: "bridge",
+			source=%P::SOURCE_NAME,
+			num_headers=%info.num_headers(),
+			justified_header=?maybe_justified_header.as_ref().map(|justified_header| &justified_header.header),
+			"Read headers. Selected finality proof for header"
+		);
+
+		Ok(match maybe_justified_header {
+			Some(justified_header) => Self::Regular(unjustified_headers, justified_header),
+			None => Self::None(unjustified_headers),
+		})
+	}
+
+	/// Returns selected mandatory header if we have seen one. Otherwise returns `None`.
+	pub fn select_mandatory(self) -> Option<JustifiedHeader<P>> {
+		match self {
+			JustifiedHeaderSelector::Mandatory(header) => Some(header),
+			_ => None,
+		}
+	}
+
+	/// Tries to improve previously selected header using ephemeral
+	/// justifications stream.
+	pub fn select(
+		self,
+		info: &SyncInfo<P>,
+		headers_to_relay: HeadersToRelay,
+		free_headers_interval: Option<P::Number>,
+		buf: &FinalityProofsBuf<P>,
+	) -> Option<JustifiedHeader<P>> {
+		let (unjustified_headers, maybe_justified_header) = match self {
+			JustifiedHeaderSelector::Mandatory(justified_header) => return Some(justified_header),
+			JustifiedHeaderSelector::Regular(unjustified_headers, justified_header) =>
+				(unjustified_headers, Some(justified_header)),
+			JustifiedHeaderSelector::None(unjustified_headers) => (unjustified_headers, None),
+		};
+
+		let mut finality_proofs_iter = buf.buf().iter().rev();
+		let mut maybe_finality_proof = finality_proofs_iter.next();
+
+		let mut unjustified_headers_iter = unjustified_headers.iter().rev();
+		let mut maybe_unjustified_header = unjustified_headers_iter.next();
+
+		while let (Some(finality_proof), Some(unjustified_header)) =
+			(maybe_finality_proof, maybe_unjustified_header)
+		{
+			match finality_proof.target_header_number().cmp(&unjustified_header.number()) {
+				Ordering::Equal
+					if need_to_relay::<P>(
+						info,
+						headers_to_relay,
+						free_headers_interval,
+						&unjustified_header,
+					) =>
+				{
+					tracing::trace!(
+						target: "bridge",
+						source=%P::SOURCE_NAME,
+						justified_header=?maybe_justified_header.as_ref().map(|justified_header| justified_header.number()),
+						target_header_number=?finality_proof.target_header_number(),
+						"Managed to improve selected finality proof."
+					);
+					return Some(JustifiedHeader {
+						header: unjustified_header.clone(),
+						proof: finality_proof.clone(),
+					});
+				},
+				Ordering::Equal => {
+					maybe_finality_proof = finality_proofs_iter.next();
+					maybe_unjustified_header = unjustified_headers_iter.next();
+				},
+				Ordering::Less => maybe_unjustified_header = unjustified_headers_iter.next(),
+				Ordering::Greater => {
+					maybe_finality_proof = finality_proofs_iter.next();
+				},
+			}
+		}
+
+		tracing::trace!(
+			target: "bridge",
+			source=%P::SOURCE_NAME,
+			justified_header=?maybe_justified_header.as_ref().map(|justified_header| justified_header.number()),
+			"Could not improve selected finality proof."
+		);
+		maybe_justified_header
+	}
+}
+
+/// Returns true if we want to relay header `header_number`.
+fn need_to_relay<P: FinalitySyncPipeline>(
+	info: &SyncInfo<P>,
+	headers_to_relay: HeadersToRelay,
+	free_headers_interval: Option<P::Number>,
+	header: &P::Header,
+) -> bool {
+	match headers_to_relay {
+		HeadersToRelay::All => true,
+		HeadersToRelay::Mandatory => header.is_mandatory(),
+		HeadersToRelay::Free =>
+			header.is_mandatory() ||
+				free_headers_interval
+					.map(|free_headers_interval| {
+						header.number().saturating_sub(info.best_number_at_target) >=
+							free_headers_interval
+					})
+					.unwrap_or(false),
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::mock::*;
+
+	#[test]
+	fn select_better_recent_finality_proof_works() {
+		let info = SyncInfo {
+			best_number_at_source: 10,
+			best_number_at_target: 5,
+			is_using_same_fork: true,
+		};
+
+		// if there are no unjustified headers, nothing is changed
+		let finality_proofs_buf =
+			FinalityProofsBuf::<TestFinalitySyncPipeline>::new(vec![TestFinalityProof(5)]);
+		let justified_header =
+			JustifiedHeader { header: TestSourceHeader(false, 2, 2), proof: TestFinalityProof(2) };
+		let selector = JustifiedHeaderSelector::Regular(vec![], justified_header.clone());
+		assert_eq!(
+			selector.select(&info, HeadersToRelay::All, None, &finality_proofs_buf),
+			Some(justified_header)
+		);
+
+		// if there are no buffered finality proofs, nothing is changed
+		let finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline>::new(vec![]);
+		let justified_header =
+			JustifiedHeader { header: TestSourceHeader(false, 2, 2), proof: TestFinalityProof(2) };
+		let selector = JustifiedHeaderSelector::Regular(
+			vec![TestSourceHeader(false, 5, 5)],
+			justified_header.clone(),
+		);
+		assert_eq!(
+			selector.select(&info, HeadersToRelay::All, None, &finality_proofs_buf),
+			Some(justified_header)
+		);
+
+		// if there's no intersection between recent finality proofs and unjustified headers,
+		// nothing is changed
+		let finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline>::new(vec![
+			TestFinalityProof(1),
+			TestFinalityProof(4),
+		]);
+		let justified_header =
+			JustifiedHeader { header: TestSourceHeader(false, 2, 2), proof: TestFinalityProof(2) };
+		let selector = JustifiedHeaderSelector::Regular(
+			vec![TestSourceHeader(false, 9, 9), TestSourceHeader(false, 10, 10)],
+			justified_header.clone(),
+		);
+		assert_eq!(
+			selector.select(&info, HeadersToRelay::All, None, &finality_proofs_buf),
+			Some(justified_header)
+		);
+
+		// if there's intersection between recent finality proofs and unjustified headers, but there
+		// are no proofs in this intersection, nothing is changed
+		let finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline>::new(vec![
+			TestFinalityProof(7),
+			TestFinalityProof(11),
+		]);
+		let justified_header =
+			JustifiedHeader { header: TestSourceHeader(false, 2, 2), proof: TestFinalityProof(2) };
+		let selector = JustifiedHeaderSelector::Regular(
+			vec![
+				TestSourceHeader(false, 8, 8),
+				TestSourceHeader(false, 9, 9),
+				TestSourceHeader(false, 10, 10),
+			],
+			justified_header.clone(),
+		);
+		assert_eq!(
+			selector.select(&info, HeadersToRelay::All, None, &finality_proofs_buf),
+			Some(justified_header)
+		);
+
+		// if there's intersection between recent finality proofs and unjustified headers and
+		// there's a proof in this intersection:
+		// - this better (last from intersection) proof is selected;
+		// - 'obsolete' unjustified headers are pruned.
+		let finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline>::new(vec![
+			TestFinalityProof(7),
+			TestFinalityProof(9),
+		]);
+		let justified_header =
+			JustifiedHeader { header: TestSourceHeader(false, 2, 2), proof: TestFinalityProof(2) };
+		let selector = JustifiedHeaderSelector::Regular(
+			vec![
+				TestSourceHeader(false, 8, 8),
+				TestSourceHeader(false, 9, 9),
+				TestSourceHeader(false, 10, 10),
+			],
+			justified_header,
+		);
+		assert_eq!(
+			selector.select(&info, HeadersToRelay::All, None, &finality_proofs_buf),
+			Some(JustifiedHeader {
+				header: TestSourceHeader(false, 9, 9),
+				proof: TestFinalityProof(9)
+			})
+		);
+
+		// when only free headers needs to be relayed and there are no free headers
+		let finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline>::new(vec![
+			TestFinalityProof(7),
+			TestFinalityProof(9),
+		]);
+		let selector = JustifiedHeaderSelector::None(vec![
+			TestSourceHeader(false, 8, 8),
+			TestSourceHeader(false, 9, 9),
+			TestSourceHeader(false, 10, 10),
+		]);
+		assert_eq!(
+			selector.select(&info, HeadersToRelay::Free, Some(7), &finality_proofs_buf),
+			None,
+		);
+
+		// when only free headers needs to be relayed, mandatory header may be selected
+		let finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline>::new(vec![
+			TestFinalityProof(6),
+			TestFinalityProof(9),
+		]);
+		let selector = JustifiedHeaderSelector::None(vec![
+			TestSourceHeader(false, 8, 8),
+			TestSourceHeader(true, 9, 9),
+			TestSourceHeader(false, 10, 10),
+		]);
+		assert_eq!(
+			selector.select(&info, HeadersToRelay::Free, Some(7), &finality_proofs_buf),
+			Some(JustifiedHeader {
+				header: TestSourceHeader(true, 9, 9),
+				proof: TestFinalityProof(9)
+			})
+		);
+
+		// when only free headers needs to be relayed and there is free header
+		let finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline>::new(vec![
+			TestFinalityProof(7),
+			TestFinalityProof(9),
+			TestFinalityProof(14),
+		]);
+		let selector = JustifiedHeaderSelector::None(vec![
+			TestSourceHeader(false, 7, 7),
+			TestSourceHeader(false, 10, 10),
+			TestSourceHeader(false, 14, 14),
+		]);
+		assert_eq!(
+			selector.select(&info, HeadersToRelay::Free, Some(7), &finality_proofs_buf),
+			Some(JustifiedHeader {
+				header: TestSourceHeader(false, 14, 14),
+				proof: TestFinalityProof(14)
+			})
+		);
+	}
+}

pezbridges/relays/finality/src/lib.rs

@@ -0,0 +1,93 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! This crate has single entrypoint to run synchronization loop that is built around finality
+//! proofs, as opposed to headers synchronization loop, which is built around headers. The headers
+//! are still submitted to the target node, but are treated as auxiliary data as we are not trying
+//! to submit all source headers to the target node.
+
+pub use crate::{
+	base::{FinalityPipeline, SourceClientBase},
+	finality_loop::{
+		metrics_prefix, run, FinalitySyncParams, HeadersToRelay, SourceClient, TargetClient,
+	},
+	finality_proofs::{FinalityProofsBuf, FinalityProofsStream},
+	sync_loop_metrics::SyncLoopMetrics,
+};
+
+use bp_header_pez_chain::ConsensusLogReader;
+use relay_utils::{FailedClient, MaybeConnectionError};
+use std::fmt::Debug;
+
+mod base;
+mod finality_loop;
+mod finality_proofs;
+mod headers;
+mod mock;
+mod sync_loop_metrics;
+
+/// Finality proofs synchronization pipeline.
+pub trait FinalitySyncPipeline: FinalityPipeline {
+	/// A reader that can extract the consensus log from the header digest and interpret it.
+	type ConsensusLogReader: ConsensusLogReader;
+	/// Type of header that we're syncing.
+	type Header: SourceHeader<Self::Hash, Self::Number, Self::ConsensusLogReader>;
+}
+
+/// Header that we're receiving from source node.
+pub trait SourceHeader<Hash, Number, Reader>: Clone + Debug + PartialEq + Send + Sync {
+	/// Returns hash of header.
+	fn hash(&self) -> Hash;
+	/// Returns number of header.
+	fn number(&self) -> Number;
+	/// Returns true if this header needs to be submitted to target node.
+	fn is_mandatory(&self) -> bool;
+}
+
+/// Error that may happen inside finality synchronization loop.
+#[derive(Debug)]
+enum Error<P: FinalitySyncPipeline, SourceError, TargetError> {
+	/// Source client request has failed with given error.
+	Source(SourceError),
+	/// Target client request has failed with given error.
+	Target(TargetError),
+	/// Finality proof for mandatory header is missing from the source node.
+	MissingMandatoryFinalityProof(P::Number),
+	/// `submit_finality_proof` transaction failed
+	ProofSubmissionTxFailed {
+		#[allow(dead_code)]
+		submitted_number: P::Number,
+		#[allow(dead_code)]
+		best_number_at_target: P::Number,
+	},
+	/// `submit_finality_proof` transaction lost
+	ProofSubmissionTxLost,
+}
+
+impl<P, SourceError, TargetError> Error<P, SourceError, TargetError>
+where
+	P: FinalitySyncPipeline,
+	SourceError: MaybeConnectionError,
+	TargetError: MaybeConnectionError,
+{
+	fn fail_if_connection_error(&self) -> Result<(), FailedClient> {
+		match *self {
+			Error::Source(ref error) if error.is_connection_error() => Err(FailedClient::Source),
+			Error::Target(ref error) if error.is_connection_error() => Err(FailedClient::Target),
+			_ => Ok(()),
+		}
+	}
+}

pezbridges/relays/finality/src/mock.rs

@@ -0,0 +1,218 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Tests for finality synchronization loop.
+
+#![cfg(test)]
+
+use crate::{
+	base::SourceClientBase,
+	finality_loop::{SourceClient, TargetClient},
+	FinalityPipeline, FinalitySyncPipeline, SourceHeader,
+};
+
+use async_trait::async_trait;
+use bp_header_pez_chain::{FinalityProof, GrandpaConsensusLogReader};
+use futures::{Stream, StreamExt};
+use parking_lot::Mutex;
+use relay_utils::{
+	relay_loop::Client as RelayClient, HeaderId, MaybeConnectionError, TrackedTransactionStatus,
+	TransactionTracker,
+};
+use std::{collections::HashMap, pin::Pin, sync::Arc};
+
+type IsMandatory = bool;
+pub type TestNumber = u64;
+type TestHash = u64;
+
+#[derive(Clone, Debug)]
+pub struct TestTransactionTracker(pub TrackedTransactionStatus<HeaderId<TestHash, TestNumber>>);
+
+impl Default for TestTransactionTracker {
+	fn default() -> TestTransactionTracker {
+		TestTransactionTracker(TrackedTransactionStatus::Finalized(Default::default()))
+	}
+}
+
+#[async_trait]
+impl TransactionTracker for TestTransactionTracker {
+	type HeaderId = HeaderId<TestHash, TestNumber>;
+
+	async fn wait(self) -> TrackedTransactionStatus<HeaderId<TestHash, TestNumber>> {
+		self.0
+	}
+}
+
+#[derive(Debug, Clone)]
+pub enum TestError {
+	NonConnection,
+}
+
+impl MaybeConnectionError for TestError {
+	fn is_connection_error(&self) -> bool {
+		false
+	}
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct TestFinalitySyncPipeline;
+
+impl FinalityPipeline for TestFinalitySyncPipeline {
+	const SOURCE_NAME: &'static str = "TestSource";
+	const TARGET_NAME: &'static str = "TestTarget";
+
+	type Hash = TestHash;
+	type Number = TestNumber;
+	type FinalityProof = TestFinalityProof;
+}
+
+impl FinalitySyncPipeline for TestFinalitySyncPipeline {
+	type ConsensusLogReader = GrandpaConsensusLogReader<TestNumber>;
+	type Header = TestSourceHeader;
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TestSourceHeader(pub IsMandatory, pub TestNumber, pub TestHash);
+
+impl SourceHeader<TestHash, TestNumber, GrandpaConsensusLogReader<TestNumber>>
+	for TestSourceHeader
+{
+	fn hash(&self) -> TestHash {
+		self.2
+	}
+
+	fn number(&self) -> TestNumber {
+		self.1
+	}
+
+	fn is_mandatory(&self) -> bool {
+		self.0
+	}
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TestFinalityProof(pub TestNumber);
+
+impl FinalityProof<TestHash, TestNumber> for TestFinalityProof {
+	fn target_header_hash(&self) -> TestHash {
+		Default::default()
+	}
+
+	fn target_header_number(&self) -> TestNumber {
+		self.0
+	}
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct ClientsData {
+	pub source_best_block_number: TestNumber,
+	pub source_headers: HashMap<TestNumber, (TestSourceHeader, Option<TestFinalityProof>)>,
+	pub source_proofs: Vec<TestFinalityProof>,
+
+	pub target_best_block_id: HeaderId<TestHash, TestNumber>,
+	pub target_headers: Vec<(TestSourceHeader, TestFinalityProof)>,
+	pub target_transaction_tracker: TestTransactionTracker,
+}
+
+#[derive(Clone)]
+pub struct TestSourceClient {
+	pub on_method_call: Arc<dyn Fn(&mut ClientsData) + Send + Sync>,
+	pub data: Arc<Mutex<ClientsData>>,
+}
+
+#[async_trait]
+impl RelayClient for TestSourceClient {
+	type Error = TestError;
+
+	async fn reconnect(&mut self) -> Result<(), TestError> {
+		unreachable!()
+	}
+}
+
+#[async_trait]
+impl SourceClientBase<TestFinalitySyncPipeline> for TestSourceClient {
+	type FinalityProofsStream = Pin<Box<dyn Stream<Item = TestFinalityProof> + 'static + Send>>;
+
+	async fn finality_proofs(&self) -> Result<Self::FinalityProofsStream, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(&mut data);
+		Ok(futures::stream::iter(data.source_proofs.clone()).boxed())
+	}
+}
+
+#[async_trait]
+impl SourceClient<TestFinalitySyncPipeline> for TestSourceClient {
+	async fn best_finalized_block_number(&self) -> Result<TestNumber, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(&mut data);
+		Ok(data.source_best_block_number)
+	}
+
+	async fn header_and_finality_proof(
+		&self,
+		number: TestNumber,
+	) -> Result<(TestSourceHeader, Option<TestFinalityProof>), TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(&mut data);
+		data.source_headers.get(&number).cloned().ok_or(TestError::NonConnection)
+	}
+}
+
+#[derive(Clone)]
+pub struct TestTargetClient {
+	pub on_method_call: Arc<dyn Fn(&mut ClientsData) + Send + Sync>,
+	pub data: Arc<Mutex<ClientsData>>,
+}
+
+#[async_trait]
+impl RelayClient for TestTargetClient {
+	type Error = TestError;
+
+	async fn reconnect(&mut self) -> Result<(), TestError> {
+		unreachable!()
+	}
+}
+
+#[async_trait]
+impl TargetClient<TestFinalitySyncPipeline> for TestTargetClient {
+	type TransactionTracker = TestTransactionTracker;
+
+	async fn best_finalized_source_block_id(
+		&self,
+	) -> Result<HeaderId<TestHash, TestNumber>, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(&mut data);
+		Ok(data.target_best_block_id)
+	}
+
+	async fn free_source_headers_interval(&self) -> Result<Option<TestNumber>, TestError> {
+		Ok(Some(3))
+	}
+
+	async fn submit_finality_proof(
+		&self,
+		header: TestSourceHeader,
+		proof: TestFinalityProof,
+		_is_free_execution_expected: bool,
+	) -> Result<TestTransactionTracker, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(&mut data);
+		data.target_best_block_id = HeaderId(header.number(), header.hash());
+		data.target_headers.push((header, proof));
+		(self.on_method_call)(&mut data);
+		Ok(data.target_transaction_tracker.clone())
+	}
+}

pezbridges/relays/finality/src/sync_loop_metrics.rs

@@ -0,0 +1,95 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Metrics for headers synchronization relay loop.
+
+use relay_utils::{
+	metrics::{metric_name, register, IntGauge, Metric, PrometheusError, Registry},
+	UniqueSaturatedInto,
+};
+
+/// Headers sync metrics.
+#[derive(Clone)]
+pub struct SyncLoopMetrics {
+	/// Best syncing header at the source.
+	best_source_block_number: IntGauge,
+	/// Best syncing header at the target.
+	best_target_block_number: IntGauge,
+	/// Flag that has `0` value when best source headers at the source node and at-target-chain
+	/// are matching and `1` otherwise.
+	using_different_forks: IntGauge,
+}
+
+impl SyncLoopMetrics {
+	/// Create and register headers loop metrics.
+	pub fn new(
+		prefix: Option<&str>,
+		at_source_chain_label: &str,
+		at_target_chain_label: &str,
+	) -> Result<Self, PrometheusError> {
+		Ok(SyncLoopMetrics {
+			best_source_block_number: IntGauge::new(
+				metric_name(prefix, &format!("best_{at_source_chain_label}_block_number")),
+				format!("Best block number at the {at_source_chain_label}"),
+			)?,
+			best_target_block_number: IntGauge::new(
+				metric_name(prefix, &format!("best_{at_target_chain_label}_block_number")),
+				format!("Best block number at the {at_target_chain_label}"),
+			)?,
+			using_different_forks: IntGauge::new(
+				metric_name(prefix, &format!("is_{at_source_chain_label}_and_{at_target_chain_label}_using_different_forks")),
+				"Whether the best finalized source block at target node is different (value 1) from the \
+				corresponding block at the source node",
+			)?,
+		})
+	}
+
+	/// Returns current value of the using-same-fork flag.
+	#[cfg(test)]
+	pub(crate) fn is_using_same_fork(&self) -> bool {
+		self.using_different_forks.get() == 0
+	}
+
+	/// Update best block number at source.
+	pub fn update_best_block_at_source<Number: UniqueSaturatedInto<u64>>(
+		&self,
+		source_best_number: Number,
+	) {
+		self.best_source_block_number.set(source_best_number.unique_saturated_into());
+	}
+
+	/// Update best block number at target.
+	pub fn update_best_block_at_target<Number: UniqueSaturatedInto<u64>>(
+		&self,
+		target_best_number: Number,
+	) {
+		self.best_target_block_number.set(target_best_number.unique_saturated_into());
+	}
+
+	/// Update using-same-fork flag.
+	pub fn update_using_same_fork(&self, using_same_fork: bool) {
+		self.using_different_forks.set((!using_same_fork).into())
+	}
+}
+
+impl Metric for SyncLoopMetrics {
+	fn register(&self, registry: &Registry) -> Result<(), PrometheusError> {
+		register(self.best_source_block_number.clone(), registry)?;
+		register(self.best_target_block_number.clone(), registry)?;
+		register(self.using_different_forks.clone(), registry)?;
+		Ok(())
+	}
+}