Reorganize relay code to make it easy to add new networks. (#813)

* Nest some crates.

* Alter command execution to make it easier to add new bridges.

* Rename sub-dirs.

* cargo fmt --all

* Address clippy.

* Update relays/substrate/src/rialto_millau/cli.rs

Co-authored-by: Hernando Castano <HCastano@users.noreply.github.com>

Co-authored-by: Hernando Castano <HCastano@users.noreply.github.com>

bridges/relays/generic/exchange/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "exchange-relay"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+
+[dependencies]
+async-std = "1.6.5"
+async-trait = "0.1.40"
+backoff = "0.2"
+futures = "0.3.5"
+log = "0.4.11"
+num-traits = "0.2"
+parking_lot = "0.11.0"
+relay-utils = { path = "../utils" }

bridges/relays/generic/exchange/src/exchange.rs

@@ -0,0 +1,916 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Relaying proofs of exchange transaction.
+
+use async_trait::async_trait;
+use relay_utils::{
+	relay_loop::Client as RelayClient, FailedClient, MaybeConnectionError, StringifiedMaybeConnectionError,
+};
+use std::{
+	fmt::{Debug, Display},
+	string::ToString,
+};
+
+/// Transaction proof pipeline.
+pub trait TransactionProofPipeline {
+	/// Name of the transaction proof source.
+	const SOURCE_NAME: &'static str;
+	/// Name of the transaction proof target.
+	const TARGET_NAME: &'static str;
+
+	/// Block type.
+	type Block: SourceBlock;
+	/// Transaction inclusion proof type.
+	type TransactionProof;
+}
+
+/// Block that is participating in exchange.
+pub trait SourceBlock {
+	/// Block hash type.
+	type Hash: Clone + Debug + Display;
+	/// Block number type.
+	type Number: Debug
+		+ Display
+		+ Clone
+		+ Copy
+		+ Into<u64>
+		+ std::cmp::Ord
+		+ std::ops::Add<Output = Self::Number>
+		+ num_traits::One;
+	/// Block transaction.
+	type Transaction: SourceTransaction;
+
+	/// Return hash of the block.
+	fn id(&self) -> relay_utils::HeaderId<Self::Hash, Self::Number>;
+	/// Return block transactions iterator.
+	fn transactions(&self) -> Vec<Self::Transaction>;
+}
+
+/// Transaction that is participating in exchange.
+pub trait SourceTransaction {
+	/// Transaction hash type.
+	type Hash: Debug + Display;
+
+	/// Return transaction hash.
+	fn hash(&self) -> Self::Hash;
+}
+
+/// Block hash for given pipeline.
+pub type BlockHashOf<P> = <<P as TransactionProofPipeline>::Block as SourceBlock>::Hash;
+
+/// Block number for given pipeline.
+pub type BlockNumberOf<P> = <<P as TransactionProofPipeline>::Block as SourceBlock>::Number;
+
+/// Transaction hash for given pipeline.
+pub type TransactionOf<P> = <<P as TransactionProofPipeline>::Block as SourceBlock>::Transaction;
+
+/// Transaction hash for given pipeline.
+pub type TransactionHashOf<P> = <TransactionOf<P> as SourceTransaction>::Hash;
+
+/// Header id.
+pub type HeaderId<P> = relay_utils::HeaderId<BlockHashOf<P>, BlockNumberOf<P>>;
+
+/// Source client API.
+#[async_trait]
+pub trait SourceClient<P: TransactionProofPipeline>: RelayClient {
+	/// Sleep until exchange-related data is (probably) updated.
+	async fn tick(&self);
+	/// Get block by hash.
+	async fn block_by_hash(&self, hash: BlockHashOf<P>) -> Result<P::Block, Self::Error>;
+	/// Get canonical block by number.
+	async fn block_by_number(&self, number: BlockNumberOf<P>) -> Result<P::Block, Self::Error>;
+	/// Return block + index where transaction has been **mined**. May return `Ok(None)` if transaction
+	/// is unknown to the source node.
+	async fn transaction_block(&self, hash: &TransactionHashOf<P>)
+		-> Result<Option<(HeaderId<P>, usize)>, Self::Error>;
+	/// Prepare transaction proof.
+	async fn transaction_proof(&self, block: &P::Block, tx_index: usize) -> Result<P::TransactionProof, Self::Error>;
+}
+
+/// Target client API.
+#[async_trait]
+pub trait TargetClient<P: TransactionProofPipeline>: RelayClient {
+	/// Sleep until exchange-related data is (probably) updated.
+	async fn tick(&self);
+	/// Returns `Ok(true)` if header is known to the target node.
+	async fn is_header_known(&self, id: &HeaderId<P>) -> Result<bool, Self::Error>;
+	/// Returns `Ok(true)` if header is finalized by the target node.
+	async fn is_header_finalized(&self, id: &HeaderId<P>) -> Result<bool, Self::Error>;
+	/// Returns best finalized header id.
+	async fn best_finalized_header_id(&self) -> Result<HeaderId<P>, Self::Error>;
+	/// Returns `Ok(true)` if transaction proof is need to be relayed.
+	async fn filter_transaction_proof(&self, proof: &P::TransactionProof) -> Result<bool, Self::Error>;
+	/// Submits transaction proof to the target node.
+	async fn submit_transaction_proof(&self, proof: P::TransactionProof) -> Result<(), Self::Error>;
+}
+
+/// Block transaction statistics.
+#[derive(Debug, Default)]
+#[cfg_attr(test, derive(PartialEq))]
+pub struct RelayedBlockTransactions {
+	/// Total number of transactions processed (either relayed or ignored) so far.
+	pub processed: usize,
+	/// Total number of transactions successfully relayed so far.
+	pub relayed: usize,
+	/// Total number of transactions that we have failed to relay so far.
+	pub failed: usize,
+}
+
+/// Relay all suitable transactions from single block.
+///
+/// If connection error occurs, returns Err with number of successfully processed transactions.
+/// If some other error occurs, it is ignored and other transactions are processed.
+///
+/// All transaction-level traces are written by this function. This function is not tracing
+/// any information about block.
+pub async fn relay_block_transactions<P: TransactionProofPipeline>(
+	source_client: &impl SourceClient<P>,
+	target_client: &impl TargetClient<P>,
+	source_block: &P::Block,
+	mut relayed_transactions: RelayedBlockTransactions,
+) -> Result<RelayedBlockTransactions, (FailedClient, RelayedBlockTransactions)> {
+	let transactions_to_process = source_block
+		.transactions()
+		.into_iter()
+		.enumerate()
+		.skip(relayed_transactions.processed);
+	for (source_tx_index, source_tx) in transactions_to_process {
+		let result = async {
+			let source_tx_id = format!("{}/{}", source_block.id().1, source_tx_index);
+			let source_tx_proof =
+				prepare_transaction_proof(source_client, &source_tx_id, source_block, source_tx_index)
+					.await
+					.map_err(|e| (FailedClient::Source, e))?;
+
+			let needs_to_be_relayed =
+				target_client
+					.filter_transaction_proof(&source_tx_proof)
+					.await
+					.map_err(|err| {
+						(
+							FailedClient::Target,
+							StringifiedMaybeConnectionError::new(
+								err.is_connection_error(),
+								format!("Transaction filtering has failed with {:?}", err),
+							),
+						)
+					})?;
+
+			if !needs_to_be_relayed {
+				return Ok(false);
+			}
+
+			relay_ready_transaction_proof(target_client, &source_tx_id, source_tx_proof)
+				.await
+				.map(|_| true)
+				.map_err(|e| (FailedClient::Target, e))
+		}
+		.await;
+
+		// We have two options here:
+		// 1) retry with the same transaction later;
+		// 2) report error and proceed with next transaction.
+		//
+		// Option#1 may seems better, but:
+		// 1) we do not track if transaction is mined (without an error) by the target node;
+		// 2) error could be irrecoverable (e.g. when block is already pruned by bridge module or tx
+		//    has invalid format) && we'll end up in infinite loop of retrying the same transaction proof.
+		//
+		// So we're going with option#2 here (the only exception are connection errors).
+		match result {
+			Ok(false) => {
+				relayed_transactions.processed += 1;
+			}
+			Ok(true) => {
+				log::info!(
+					target: "bridge",
+					"{} transaction {} proof has been successfully submitted to {} node",
+					P::SOURCE_NAME,
+					source_tx.hash(),
+					P::TARGET_NAME,
+				);
+
+				relayed_transactions.processed += 1;
+				relayed_transactions.relayed += 1;
+			}
+			Err((failed_client, err)) => {
+				log::error!(
+					target: "bridge",
+					"Error relaying {} transaction {} proof to {} node: {}. {}",
+					P::SOURCE_NAME,
+					source_tx.hash(),
+					P::TARGET_NAME,
+					err.to_string(),
+					if err.is_connection_error() {
+						"Going to retry after delay..."
+					} else {
+						"You may need to submit proof of this transaction manually"
+					},
+				);
+
+				if err.is_connection_error() {
+					return Err((failed_client, relayed_transactions));
+				}
+
+				relayed_transactions.processed += 1;
+				relayed_transactions.failed += 1;
+			}
+		}
+	}
+
+	Ok(relayed_transactions)
+}
+
+/// Relay single transaction proof.
+pub async fn relay_single_transaction_proof<P: TransactionProofPipeline>(
+	source_client: &impl SourceClient<P>,
+	target_client: &impl TargetClient<P>,
+	source_tx_hash: TransactionHashOf<P>,
+) -> Result<(), String> {
+	// wait for transaction and header on source node
+	let (source_header_id, source_tx_index) = wait_transaction_mined(source_client, &source_tx_hash).await?;
+	let source_block = source_client.block_by_hash(source_header_id.1.clone()).await;
+	let source_block = source_block.map_err(|err| {
+		format!(
+			"Error retrieving block {} from {} node: {:?}",
+			source_header_id.1,
+			P::SOURCE_NAME,
+			err,
+		)
+	})?;
+
+	// wait for transaction and header on target node
+	wait_header_imported(target_client, &source_header_id).await?;
+	wait_header_finalized(target_client, &source_header_id).await?;
+
+	// and finally - prepare and submit transaction proof to target node
+	let source_tx_id = format!("{}", source_tx_hash);
+	relay_ready_transaction_proof(
+		target_client,
+		&source_tx_id,
+		prepare_transaction_proof(source_client, &source_tx_id, &source_block, source_tx_index)
+			.await
+			.map_err(|err| err.to_string())?,
+	)
+	.await
+	.map_err(|err| err.to_string())
+}
+
+/// Prepare transaction proof.
+async fn prepare_transaction_proof<P: TransactionProofPipeline>(
+	source_client: &impl SourceClient<P>,
+	source_tx_id: &str,
+	source_block: &P::Block,
+	source_tx_index: usize,
+) -> Result<P::TransactionProof, StringifiedMaybeConnectionError> {
+	source_client
+		.transaction_proof(source_block, source_tx_index)
+		.await
+		.map_err(|err| {
+			StringifiedMaybeConnectionError::new(
+				err.is_connection_error(),
+				format!(
+					"Error building transaction {} proof on {} node: {:?}",
+					source_tx_id,
+					P::SOURCE_NAME,
+					err,
+				),
+			)
+		})
+}
+
+/// Relay prepared proof of transaction.
+async fn relay_ready_transaction_proof<P: TransactionProofPipeline>(
+	target_client: &impl TargetClient<P>,
+	source_tx_id: &str,
+	source_tx_proof: P::TransactionProof,
+) -> Result<(), StringifiedMaybeConnectionError> {
+	target_client
+		.submit_transaction_proof(source_tx_proof)
+		.await
+		.map_err(|err| {
+			StringifiedMaybeConnectionError::new(
+				err.is_connection_error(),
+				format!(
+					"Error submitting transaction {} proof to {} node: {:?}",
+					source_tx_id,
+					P::TARGET_NAME,
+					err,
+				),
+			)
+		})
+}
+
+/// Wait until transaction is mined by source node.
+async fn wait_transaction_mined<P: TransactionProofPipeline>(
+	source_client: &impl SourceClient<P>,
+	source_tx_hash: &TransactionHashOf<P>,
+) -> Result<(HeaderId<P>, usize), String> {
+	loop {
+		let source_header_and_tx = source_client.transaction_block(&source_tx_hash).await.map_err(|err| {
+			format!(
+				"Error retrieving transaction {} from {} node: {:?}",
+				source_tx_hash,
+				P::SOURCE_NAME,
+				err,
+			)
+		})?;
+		match source_header_and_tx {
+			Some((source_header_id, source_tx)) => {
+				log::info!(
+					target: "bridge",
+					"Transaction {} is retrieved from {} node. Continuing...",
+					source_tx_hash,
+					P::SOURCE_NAME,
+				);
+
+				return Ok((source_header_id, source_tx));
+			}
+			None => {
+				log::info!(
+					target: "bridge",
+					"Waiting for transaction {} to be mined by {} node...",
+					source_tx_hash,
+					P::SOURCE_NAME,
+				);
+
+				source_client.tick().await;
+			}
+		}
+	}
+}
+
+/// Wait until target node imports required header.
+async fn wait_header_imported<P: TransactionProofPipeline>(
+	target_client: &impl TargetClient<P>,
+	source_header_id: &HeaderId<P>,
+) -> Result<(), String> {
+	loop {
+		let is_header_known = target_client.is_header_known(&source_header_id).await.map_err(|err| {
+			format!(
+				"Failed to check existence of header {}/{} on {} node: {:?}",
+				source_header_id.0,
+				source_header_id.1,
+				P::TARGET_NAME,
+				err,
+			)
+		})?;
+		match is_header_known {
+			true => {
+				log::info!(
+					target: "bridge",
+					"Header {}/{} is known to {} node. Continuing.",
+					source_header_id.0,
+					source_header_id.1,
+					P::TARGET_NAME,
+				);
+
+				return Ok(());
+			}
+			false => {
+				log::info!(
+					target: "bridge",
+					"Waiting for header {}/{} to be imported by {} node...",
+					source_header_id.0,
+					source_header_id.1,
+					P::TARGET_NAME,
+				);
+
+				target_client.tick().await;
+			}
+		}
+	}
+}
+
+/// Wait until target node finalizes required header.
+async fn wait_header_finalized<P: TransactionProofPipeline>(
+	target_client: &impl TargetClient<P>,
+	source_header_id: &HeaderId<P>,
+) -> Result<(), String> {
+	loop {
+		let is_header_finalized = target_client
+			.is_header_finalized(&source_header_id)
+			.await
+			.map_err(|err| {
+				format!(
+					"Failed to check finality of header {}/{} on {} node: {:?}",
+					source_header_id.0,
+					source_header_id.1,
+					P::TARGET_NAME,
+					err,
+				)
+			})?;
+		match is_header_finalized {
+			true => {
+				log::info!(
+					target: "bridge",
+					"Header {}/{} is finalizd by {} node. Continuing.",
+					source_header_id.0,
+					source_header_id.1,
+					P::TARGET_NAME,
+				);
+
+				return Ok(());
+			}
+			false => {
+				log::info!(
+					target: "bridge",
+					"Waiting for header {}/{} to be finalized by {} node...",
+					source_header_id.0,
+					source_header_id.1,
+					P::TARGET_NAME,
+				);
+
+				target_client.tick().await;
+			}
+		}
+	}
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+	use super::*;
+
+	use parking_lot::Mutex;
+	use relay_utils::HeaderId;
+	use std::{
+		collections::{HashMap, HashSet},
+		sync::Arc,
+	};
+
+	pub fn test_block_id() -> TestHeaderId {
+		HeaderId(1, 1)
+	}
+
+	pub fn test_next_block_id() -> TestHeaderId {
+		HeaderId(2, 2)
+	}
+
+	pub fn test_transaction_hash(tx_index: u64) -> TestTransactionHash {
+		200 + tx_index
+	}
+
+	pub fn test_transaction(tx_index: u64) -> TestTransaction {
+		TestTransaction(test_transaction_hash(tx_index))
+	}
+
+	pub fn test_block() -> TestBlock {
+		TestBlock(test_block_id(), vec![test_transaction(0)])
+	}
+
+	pub fn test_next_block() -> TestBlock {
+		TestBlock(test_next_block_id(), vec![test_transaction(1)])
+	}
+
+	pub type TestBlockNumber = u64;
+	pub type TestBlockHash = u64;
+	pub type TestTransactionHash = u64;
+	pub type TestHeaderId = HeaderId<TestBlockHash, TestBlockNumber>;
+
+	#[derive(Debug, Clone, PartialEq)]
+	pub struct TestError(pub bool);
+
+	impl MaybeConnectionError for TestError {
+		fn is_connection_error(&self) -> bool {
+			self.0
+		}
+	}
+
+	pub struct TestTransactionProofPipeline;
+
+	impl TransactionProofPipeline for TestTransactionProofPipeline {
+		const SOURCE_NAME: &'static str = "TestSource";
+		const TARGET_NAME: &'static str = "TestTarget";
+
+		type Block = TestBlock;
+		type TransactionProof = TestTransactionProof;
+	}
+
+	#[derive(Debug, Clone)]
+	pub struct TestBlock(pub TestHeaderId, pub Vec<TestTransaction>);
+
+	impl SourceBlock for TestBlock {
+		type Hash = TestBlockHash;
+		type Number = TestBlockNumber;
+		type Transaction = TestTransaction;
+
+		fn id(&self) -> TestHeaderId {
+			self.0
+		}
+
+		fn transactions(&self) -> Vec<TestTransaction> {
+			self.1.clone()
+		}
+	}
+
+	#[derive(Debug, Clone)]
+	pub struct TestTransaction(pub TestTransactionHash);
+
+	impl SourceTransaction for TestTransaction {
+		type Hash = TestTransactionHash;
+
+		fn hash(&self) -> Self::Hash {
+			self.0
+		}
+	}
+
+	#[derive(Debug, Clone, PartialEq)]
+	pub struct TestTransactionProof(pub TestTransactionHash);
+
+	#[derive(Clone)]
+	pub struct TestTransactionsSource {
+		pub on_tick: Arc<dyn Fn(&mut TestTransactionsSourceData) + Send + Sync>,
+		pub data: Arc<Mutex<TestTransactionsSourceData>>,
+	}
+
+	pub struct TestTransactionsSourceData {
+		pub block: Result<TestBlock, TestError>,
+		pub transaction_block: Result<Option<(TestHeaderId, usize)>, TestError>,
+		pub proofs_to_fail: HashMap<TestTransactionHash, TestError>,
+	}
+
+	impl TestTransactionsSource {
+		pub fn new(on_tick: Box<dyn Fn(&mut TestTransactionsSourceData) + Send + Sync>) -> Self {
+			Self {
+				on_tick: Arc::new(on_tick),
+				data: Arc::new(Mutex::new(TestTransactionsSourceData {
+					block: Ok(test_block()),
+					transaction_block: Ok(Some((test_block_id(), 0))),
+					proofs_to_fail: HashMap::new(),
+				})),
+			}
+		}
+	}
+
+	#[async_trait]
+	impl RelayClient for TestTransactionsSource {
+		type Error = TestError;
+
+		async fn reconnect(&mut self) -> Result<(), TestError> {
+			Ok(())
+		}
+	}
+
+	#[async_trait]
+	impl SourceClient<TestTransactionProofPipeline> for TestTransactionsSource {
+		async fn tick(&self) {
+			(self.on_tick)(&mut *self.data.lock())
+		}
+
+		async fn block_by_hash(&self, _: TestBlockHash) -> Result<TestBlock, TestError> {
+			self.data.lock().block.clone()
+		}
+
+		async fn block_by_number(&self, _: TestBlockNumber) -> Result<TestBlock, TestError> {
+			self.data.lock().block.clone()
+		}
+
+		async fn transaction_block(&self, _: &TestTransactionHash) -> Result<Option<(TestHeaderId, usize)>, TestError> {
+			self.data.lock().transaction_block.clone()
+		}
+
+		async fn transaction_proof(&self, block: &TestBlock, index: usize) -> Result<TestTransactionProof, TestError> {
+			let tx_hash = block.1[index].hash();
+			let proof_error = self.data.lock().proofs_to_fail.get(&tx_hash).cloned();
+			if let Some(err) = proof_error {
+				return Err(err);
+			}
+
+			Ok(TestTransactionProof(tx_hash))
+		}
+	}
+
+	#[derive(Clone)]
+	pub struct TestTransactionsTarget {
+		pub on_tick: Arc<dyn Fn(&mut TestTransactionsTargetData) + Send + Sync>,
+		pub data: Arc<Mutex<TestTransactionsTargetData>>,
+	}
+
+	pub struct TestTransactionsTargetData {
+		pub is_header_known: Result<bool, TestError>,
+		pub is_header_finalized: Result<bool, TestError>,
+		pub best_finalized_header_id: Result<TestHeaderId, TestError>,
+		pub transactions_to_accept: HashSet<TestTransactionHash>,
+		pub submitted_proofs: Vec<TestTransactionProof>,
+	}
+
+	impl TestTransactionsTarget {
+		pub fn new(on_tick: Box<dyn Fn(&mut TestTransactionsTargetData) + Send + Sync>) -> Self {
+			Self {
+				on_tick: Arc::new(on_tick),
+				data: Arc::new(Mutex::new(TestTransactionsTargetData {
+					is_header_known: Ok(true),
+					is_header_finalized: Ok(true),
+					best_finalized_header_id: Ok(test_block_id()),
+					transactions_to_accept: vec![test_transaction_hash(0)].into_iter().collect(),
+					submitted_proofs: Vec::new(),
+				})),
+			}
+		}
+	}
+
+	#[async_trait]
+	impl RelayClient for TestTransactionsTarget {
+		type Error = TestError;
+
+		async fn reconnect(&mut self) -> Result<(), TestError> {
+			Ok(())
+		}
+	}
+
+	#[async_trait]
+	impl TargetClient<TestTransactionProofPipeline> for TestTransactionsTarget {
+		async fn tick(&self) {
+			(self.on_tick)(&mut *self.data.lock())
+		}
+
+		async fn is_header_known(&self, _: &TestHeaderId) -> Result<bool, TestError> {
+			self.data.lock().is_header_known.clone()
+		}
+
+		async fn is_header_finalized(&self, _: &TestHeaderId) -> Result<bool, TestError> {
+			self.data.lock().is_header_finalized.clone()
+		}
+
+		async fn best_finalized_header_id(&self) -> Result<TestHeaderId, TestError> {
+			self.data.lock().best_finalized_header_id.clone()
+		}
+
+		async fn filter_transaction_proof(&self, proof: &TestTransactionProof) -> Result<bool, TestError> {
+			Ok(self.data.lock().transactions_to_accept.contains(&proof.0))
+		}
+
+		async fn submit_transaction_proof(&self, proof: TestTransactionProof) -> Result<(), TestError> {
+			self.data.lock().submitted_proofs.push(proof);
+			Ok(())
+		}
+	}
+
+	fn ensure_relay_single_success(source: &TestTransactionsSource, target: &TestTransactionsTarget) {
+		assert_eq!(
+			async_std::task::block_on(relay_single_transaction_proof(source, target, test_transaction_hash(0),)),
+			Ok(()),
+		);
+		assert_eq!(
+			target.data.lock().submitted_proofs,
+			vec![TestTransactionProof(test_transaction_hash(0))],
+		);
+	}
+
+	fn ensure_relay_single_failure(source: TestTransactionsSource, target: TestTransactionsTarget) {
+		assert!(async_std::task::block_on(relay_single_transaction_proof(
+			&source,
+			&target,
+			test_transaction_hash(0),
+		))
+		.is_err(),);
+		assert!(target.data.lock().submitted_proofs.is_empty());
+	}
+
+	#[test]
+	fn ready_transaction_proof_relayed_immediately() {
+		let source = TestTransactionsSource::new(Box::new(|_| unreachable!("no ticks allowed")));
+		let target = TestTransactionsTarget::new(Box::new(|_| unreachable!("no ticks allowed")));
+		ensure_relay_single_success(&source, &target)
+	}
+
+	#[test]
+	fn relay_transaction_proof_waits_for_transaction_to_be_mined() {
+		let source = TestTransactionsSource::new(Box::new(|source_data| {
+			assert_eq!(source_data.transaction_block, Ok(None));
+			source_data.transaction_block = Ok(Some((test_block_id(), 0)));
+		}));
+		let target = TestTransactionsTarget::new(Box::new(|_| unreachable!("no ticks allowed")));
+
+		// transaction is not yet mined, but will be available after first wait (tick)
+		source.data.lock().transaction_block = Ok(None);
+
+		ensure_relay_single_success(&source, &target)
+	}
+
+	#[test]
+	fn relay_transaction_fails_when_transaction_retrieval_fails() {
+		let source = TestTransactionsSource::new(Box::new(|_| unreachable!("no ticks allowed")));
+		let target = TestTransactionsTarget::new(Box::new(|_| unreachable!("no ticks allowed")));
+
+		source.data.lock().transaction_block = Err(TestError(false));
+
+		ensure_relay_single_failure(source, target)
+	}
+
+	#[test]
+	fn relay_transaction_fails_when_proof_retrieval_fails() {
+		let source = TestTransactionsSource::new(Box::new(|_| unreachable!("no ticks allowed")));
+		let target = TestTransactionsTarget::new(Box::new(|_| unreachable!("no ticks allowed")));
+
+		source
+			.data
+			.lock()
+			.proofs_to_fail
+			.insert(test_transaction_hash(0), TestError(false));
+
+		ensure_relay_single_failure(source, target)
+	}
+
+	#[test]
+	fn relay_transaction_proof_waits_for_header_to_be_imported() {
+		let source = TestTransactionsSource::new(Box::new(|_| unreachable!("no ticks allowed")));
+		let target = TestTransactionsTarget::new(Box::new(|target_data| {
+			assert_eq!(target_data.is_header_known, Ok(false));
+			target_data.is_header_known = Ok(true);
+		}));
+
+		// header is not yet imported, but will be available after first wait (tick)
+		target.data.lock().is_header_known = Ok(false);
+
+		ensure_relay_single_success(&source, &target)
+	}
+
+	#[test]
+	fn relay_transaction_proof_fails_when_is_header_known_fails() {
+		let source = TestTransactionsSource::new(Box::new(|_| unreachable!("no ticks allowed")));
+		let target = TestTransactionsTarget::new(Box::new(|_| unreachable!("no ticks allowed")));
+
+		target.data.lock().is_header_known = Err(TestError(false));
+
+		ensure_relay_single_failure(source, target)
+	}
+
+	#[test]
+	fn relay_transaction_proof_waits_for_header_to_be_finalized() {
+		let source = TestTransactionsSource::new(Box::new(|_| unreachable!("no ticks allowed")));
+		let target = TestTransactionsTarget::new(Box::new(|target_data| {
+			assert_eq!(target_data.is_header_finalized, Ok(false));
+			target_data.is_header_finalized = Ok(true);
+		}));
+
+		// header is not yet finalized, but will be available after first wait (tick)
+		target.data.lock().is_header_finalized = Ok(false);
+
+		ensure_relay_single_success(&source, &target)
+	}
+
+	#[test]
+	fn relay_transaction_proof_fails_when_is_header_finalized_fails() {
+		let source = TestTransactionsSource::new(Box::new(|_| unreachable!("no ticks allowed")));
+		let target = TestTransactionsTarget::new(Box::new(|_| unreachable!("no ticks allowed")));
+
+		target.data.lock().is_header_finalized = Err(TestError(false));
+
+		ensure_relay_single_failure(source, target)
+	}
+
+	#[test]
+	fn relay_transaction_proof_fails_when_target_node_rejects_proof() {
+		let source = TestTransactionsSource::new(Box::new(|_| unreachable!("no ticks allowed")));
+		let target = TestTransactionsTarget::new(Box::new(|_| unreachable!("no ticks allowed")));
+
+		target
+			.data
+			.lock()
+			.transactions_to_accept
+			.remove(&test_transaction_hash(0));
+
+		ensure_relay_single_success(&source, &target)
+	}
+
+	fn test_relay_block_transactions(
+		source: &TestTransactionsSource,
+		target: &TestTransactionsTarget,
+		pre_relayed: RelayedBlockTransactions,
+	) -> Result<RelayedBlockTransactions, RelayedBlockTransactions> {
+		async_std::task::block_on(relay_block_transactions(
+			source,
+			target,
+			&TestBlock(
+				test_block_id(),
+				vec![test_transaction(0), test_transaction(1), test_transaction(2)],
+			),
+			pre_relayed,
+		))
+		.map_err(|(_, transactions)| transactions)
+	}
+
+	#[test]
+	fn relay_block_transactions_process_all_transactions() {
+		let source = TestTransactionsSource::new(Box::new(|_| unreachable!("no ticks allowed")));
+		let target = TestTransactionsTarget::new(Box::new(|_| unreachable!("no ticks allowed")));
+
+		// let's only accept tx#1
+		target
+			.data
+			.lock()
+			.transactions_to_accept
+			.remove(&test_transaction_hash(0));
+		target
+			.data
+			.lock()
+			.transactions_to_accept
+			.insert(test_transaction_hash(1));
+
+		let relayed_transactions = test_relay_block_transactions(&source, &target, Default::default());
+		assert_eq!(
+			relayed_transactions,
+			Ok(RelayedBlockTransactions {
+				processed: 3,
+				relayed: 1,
+				failed: 0,
+			}),
+		);
+		assert_eq!(
+			target.data.lock().submitted_proofs,
+			vec![TestTransactionProof(test_transaction_hash(1))],
+		);
+	}
+
+	#[test]
+	fn relay_block_transactions_ignores_transaction_failure() {
+		let source = TestTransactionsSource::new(Box::new(|_| unreachable!("no ticks allowed")));
+		let target = TestTransactionsTarget::new(Box::new(|_| unreachable!("no ticks allowed")));
+
+		// let's reject proof for tx#0
+		source
+			.data
+			.lock()
+			.proofs_to_fail
+			.insert(test_transaction_hash(0), TestError(false));
+
+		let relayed_transactions = test_relay_block_transactions(&source, &target, Default::default());
+		assert_eq!(
+			relayed_transactions,
+			Ok(RelayedBlockTransactions {
+				processed: 3,
+				relayed: 0,
+				failed: 1,
+			}),
+		);
+		assert_eq!(target.data.lock().submitted_proofs, vec![],);
+	}
+
+	#[test]
+	fn relay_block_transactions_fails_on_connection_error() {
+		let source = TestTransactionsSource::new(Box::new(|_| unreachable!("no ticks allowed")));
+		let target = TestTransactionsTarget::new(Box::new(|_| unreachable!("no ticks allowed")));
+
+		// fail with connection error when preparing proof for tx#1
+		source
+			.data
+			.lock()
+			.proofs_to_fail
+			.insert(test_transaction_hash(1), TestError(true));
+
+		let relayed_transactions = test_relay_block_transactions(&source, &target, Default::default());
+		assert_eq!(
+			relayed_transactions,
+			Err(RelayedBlockTransactions {
+				processed: 1,
+				relayed: 1,
+				failed: 0,
+			}),
+		);
+		assert_eq!(
+			target.data.lock().submitted_proofs,
+			vec![TestTransactionProof(test_transaction_hash(0))],
+		);
+
+		// now do not fail on tx#2
+		source.data.lock().proofs_to_fail.clear();
+		// and also relay tx#3
+		target
+			.data
+			.lock()
+			.transactions_to_accept
+			.insert(test_transaction_hash(2));
+
+		let relayed_transactions = test_relay_block_transactions(&source, &target, relayed_transactions.unwrap_err());
+		assert_eq!(
+			relayed_transactions,
+			Ok(RelayedBlockTransactions {
+				processed: 3,
+				relayed: 2,
+				failed: 0,
+			}),
+		);
+		assert_eq!(
+			target.data.lock().submitted_proofs,
+			vec![
+				TestTransactionProof(test_transaction_hash(0)),
+				TestTransactionProof(test_transaction_hash(2))
+			],
+		);
+	}
+}

bridges/relays/generic/exchange/src/exchange_loop.rs

@@ -0,0 +1,331 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Relaying proofs of exchange transactions.
+
+use crate::exchange::{
+	relay_block_transactions, BlockNumberOf, RelayedBlockTransactions, SourceClient, TargetClient,
+	TransactionProofPipeline,
+};
+use crate::exchange_loop_metrics::ExchangeLoopMetrics;
+
+use backoff::backoff::Backoff;
+use futures::{future::FutureExt, select};
+use num_traits::One;
+use relay_utils::{
+	metrics::{start as metrics_start, GlobalMetrics, MetricsParams},
+	retry_backoff, FailedClient, MaybeConnectionError,
+};
+use std::future::Future;
+
+/// Transactions proofs relay state.
+#[derive(Debug)]
+pub struct TransactionProofsRelayState<BlockNumber> {
+	/// Number of last header we have processed so far.
+	pub best_processed_header_number: BlockNumber,
+}
+
+/// Transactions proofs relay storage.
+pub trait TransactionProofsRelayStorage: Clone {
+	/// Associated block number.
+	type BlockNumber;
+
+	/// Get relay state.
+	fn state(&self) -> TransactionProofsRelayState<Self::BlockNumber>;
+	/// Update relay state.
+	fn set_state(&mut self, state: &TransactionProofsRelayState<Self::BlockNumber>);
+}
+
+/// In-memory storage for auto-relay loop.
+#[derive(Debug, Clone)]
+pub struct InMemoryStorage<BlockNumber> {
+	best_processed_header_number: BlockNumber,
+}
+
+impl<BlockNumber> InMemoryStorage<BlockNumber> {
+	/// Created new in-memory storage with given best processed block number.
+	pub fn new(best_processed_header_number: BlockNumber) -> Self {
+		InMemoryStorage {
+			best_processed_header_number,
+		}
+	}
+}
+
+impl<BlockNumber: Clone + Copy> TransactionProofsRelayStorage for InMemoryStorage<BlockNumber> {
+	type BlockNumber = BlockNumber;
+
+	fn state(&self) -> TransactionProofsRelayState<BlockNumber> {
+		TransactionProofsRelayState {
+			best_processed_header_number: self.best_processed_header_number,
+		}
+	}
+
+	fn set_state(&mut self, state: &TransactionProofsRelayState<BlockNumber>) {
+		self.best_processed_header_number = state.best_processed_header_number;
+	}
+}
+
+/// Run proofs synchronization.
+pub fn run<P: TransactionProofPipeline>(
+	storage: impl TransactionProofsRelayStorage<BlockNumber = BlockNumberOf<P>>,
+	source_client: impl SourceClient<P>,
+	target_client: impl TargetClient<P>,
+	metrics_params: Option<MetricsParams>,
+	exit_signal: impl Future<Output = ()>,
+) {
+	let exit_signal = exit_signal.shared();
+	let metrics_global = GlobalMetrics::default();
+	let metrics_exch = ExchangeLoopMetrics::default();
+	let metrics_enabled = metrics_params.is_some();
+	metrics_start(
+		format!("{}_to_{}_Exchange", P::SOURCE_NAME, P::TARGET_NAME),
+		metrics_params,
+		&metrics_global,
+		&metrics_exch,
+	);
+
+	relay_utils::relay_loop::run(
+		relay_utils::relay_loop::RECONNECT_DELAY,
+		source_client,
+		target_client,
+		|source_client, target_client| {
+			run_until_connection_lost(
+				storage.clone(),
+				source_client,
+				target_client,
+				if metrics_enabled {
+					Some(metrics_global.clone())
+				} else {
+					None
+				},
+				if metrics_enabled {
+					Some(metrics_exch.clone())
+				} else {
+					None
+				},
+				exit_signal.clone(),
+			)
+		},
+	);
+}
+
+/// Run proofs synchronization.
+async fn run_until_connection_lost<P: TransactionProofPipeline>(
+	mut storage: impl TransactionProofsRelayStorage<BlockNumber = BlockNumberOf<P>>,
+	source_client: impl SourceClient<P>,
+	target_client: impl TargetClient<P>,
+	metrics_global: Option<GlobalMetrics>,
+	metrics_exch: Option<ExchangeLoopMetrics>,
+	exit_signal: impl Future<Output = ()>,
+) -> Result<(), FailedClient> {
+	let mut retry_backoff = retry_backoff();
+	let mut state = storage.state();
+	let mut current_finalized_block = None;
+
+	let exit_signal = exit_signal.fuse();
+
+	futures::pin_mut!(exit_signal);
+
+	loop {
+		let iteration_result = run_loop_iteration(
+			&mut storage,
+			&source_client,
+			&target_client,
+			&mut state,
+			&mut current_finalized_block,
+			metrics_exch.as_ref(),
+		)
+		.await;
+
+		if let Some(ref metrics_global) = metrics_global {
+			metrics_global.update().await;
+		}
+
+		if let Err((is_connection_error, failed_client)) = iteration_result {
+			if is_connection_error {
+				return Err(failed_client);
+			}
+
+			let retry_timeout = retry_backoff
+				.next_backoff()
+				.unwrap_or(relay_utils::relay_loop::RECONNECT_DELAY);
+			select! {
+				_ = async_std::task::sleep(retry_timeout).fuse() => {},
+				_ = exit_signal => return Ok(()),
+			}
+		} else {
+			retry_backoff.reset();
+
+			select! {
+				_ = source_client.tick().fuse() => {},
+				_ = exit_signal => return Ok(()),
+			}
+		}
+	}
+}
+
+/// Run exchange loop until we need to break.
+async fn run_loop_iteration<P: TransactionProofPipeline>(
+	storage: &mut impl TransactionProofsRelayStorage<BlockNumber = BlockNumberOf<P>>,
+	source_client: &impl SourceClient<P>,
+	target_client: &impl TargetClient<P>,
+	state: &mut TransactionProofsRelayState<BlockNumberOf<P>>,
+	current_finalized_block: &mut Option<(P::Block, RelayedBlockTransactions)>,
+	exchange_loop_metrics: Option<&ExchangeLoopMetrics>,
+) -> Result<(), (bool, FailedClient)> {
+	let best_finalized_header_id = match target_client.best_finalized_header_id().await {
+		Ok(best_finalized_header_id) => {
+			log::debug!(
+				target: "bridge",
+				"Got best finalized {} block from {} node: {:?}",
+				P::SOURCE_NAME,
+				P::TARGET_NAME,
+				best_finalized_header_id,
+			);
+
+			best_finalized_header_id
+		}
+		Err(err) => {
+			log::error!(
+				target: "bridge",
+				"Failed to retrieve best {} header id from {} node: {:?}. Going to retry...",
+				P::SOURCE_NAME,
+				P::TARGET_NAME,
+				err,
+			);
+
+			return Err((err.is_connection_error(), FailedClient::Target));
+		}
+	};
+
+	loop {
+		// if we already have some finalized block body, try to relay its transactions
+		if let Some((block, relayed_transactions)) = current_finalized_block.take() {
+			let result = relay_block_transactions(source_client, target_client, &block, relayed_transactions).await;
+
+			match result {
+				Ok(relayed_transactions) => {
+					log::info!(
+						target: "bridge",
+						"Relay has processed {} block #{}. Total/Relayed/Failed transactions: {}/{}/{}",
+						P::SOURCE_NAME,
+						state.best_processed_header_number,
+						relayed_transactions.processed,
+						relayed_transactions.relayed,
+						relayed_transactions.failed,
+					);
+
+					state.best_processed_header_number = state.best_processed_header_number + One::one();
+					storage.set_state(state);
+
+					if let Some(ref exchange_loop_metrics) = exchange_loop_metrics {
+						exchange_loop_metrics.update::<P>(
+							state.best_processed_header_number,
+							best_finalized_header_id.0,
+							relayed_transactions,
+						);
+					}
+
+					// we have just updated state => proceed to next block retrieval
+				}
+				Err((failed_client, relayed_transactions)) => {
+					*current_finalized_block = Some((block, relayed_transactions));
+					return Err((true, failed_client));
+				}
+			}
+		}
+
+		// we may need to retrieve finalized block body from source node
+		if best_finalized_header_id.0 > state.best_processed_header_number {
+			let next_block_number = state.best_processed_header_number + One::one();
+			let result = source_client.block_by_number(next_block_number).await;
+
+			match result {
+				Ok(block) => {
+					*current_finalized_block = Some((block, RelayedBlockTransactions::default()));
+
+					// we have received new finalized block => go back to relay its transactions
+					continue;
+				}
+				Err(err) => {
+					log::error!(
+						target: "bridge",
+						"Failed to retrieve canonical block #{} from {} node: {:?}. Going to retry...",
+						next_block_number,
+						P::SOURCE_NAME,
+						err,
+					);
+
+					return Err((err.is_connection_error(), FailedClient::Source));
+				}
+			}
+		}
+
+		// there are no any transactions we need to relay => wait for new data
+		return Ok(());
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::exchange::tests::{
+		test_next_block, test_next_block_id, test_transaction_hash, TestTransactionProof, TestTransactionsSource,
+		TestTransactionsTarget,
+	};
+	use futures::{future::FutureExt, stream::StreamExt};
+
+	#[test]
+	fn exchange_loop_is_able_to_relay_proofs() {
+		let storage = InMemoryStorage {
+			best_processed_header_number: 0,
+		};
+		let target = TestTransactionsTarget::new(Box::new(|_| unreachable!("no target ticks allowed")));
+		let target_data = target.data.clone();
+		let (exit_sender, exit_receiver) = futures::channel::mpsc::unbounded();
+
+		let source = TestTransactionsSource::new(Box::new(move |data| {
+			let transaction1_relayed = target_data
+				.lock()
+				.submitted_proofs
+				.contains(&TestTransactionProof(test_transaction_hash(0)));
+			let transaction2_relayed = target_data
+				.lock()
+				.submitted_proofs
+				.contains(&TestTransactionProof(test_transaction_hash(1)));
+			match (transaction1_relayed, transaction2_relayed) {
+				(true, true) => exit_sender.unbounded_send(()).unwrap(),
+				(true, false) => {
+					data.block = Ok(test_next_block());
+					target_data.lock().best_finalized_header_id = Ok(test_next_block_id());
+					target_data
+						.lock()
+						.transactions_to_accept
+						.insert(test_transaction_hash(1));
+				}
+				_ => (),
+			}
+		}));
+
+		run(
+			storage,
+			source,
+			target,
+			None,
+			exit_receiver.into_future().map(|(_, _)| ()),
+		);
+	}
+}

bridges/relays/generic/exchange/src/exchange_loop_metrics.rs

@@ -0,0 +1,88 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Metrics for currency-exchange relay loop.
+
+use crate::exchange::{BlockNumberOf, RelayedBlockTransactions, TransactionProofPipeline};
+use relay_utils::metrics::{register, Counter, CounterVec, GaugeVec, Metrics, Opts, Registry, U64};
+
+/// Exchange transactions relay metrics.
+#[derive(Clone)]
+pub struct ExchangeLoopMetrics {
+	/// Best finalized block numbers - "processed" and "known".
+	best_block_numbers: GaugeVec<U64>,
+	/// Number of processed blocks ("total").
+	processed_blocks: Counter<U64>,
+	/// Number of processed transactions ("total", "relayed" and "failed").
+	processed_transactions: CounterVec<U64>,
+}
+
+impl Metrics for ExchangeLoopMetrics {
+	fn register(&self, registry: &Registry) -> Result<(), String> {
+		register(self.best_block_numbers.clone(), registry).map_err(|e| e.to_string())?;
+		register(self.processed_blocks.clone(), registry).map_err(|e| e.to_string())?;
+		register(self.processed_transactions.clone(), registry).map_err(|e| e.to_string())?;
+		Ok(())
+	}
+}
+
+impl Default for ExchangeLoopMetrics {
+	fn default() -> Self {
+		ExchangeLoopMetrics {
+			best_block_numbers: GaugeVec::new(
+				Opts::new("best_block_numbers", "Best finalized block numbers"),
+				&["type"],
+			)
+			.expect("metric is static and thus valid; qed"),
+			processed_blocks: Counter::new("processed_blocks", "Total number of processed blocks")
+				.expect("metric is static and thus valid; qed"),
+			processed_transactions: CounterVec::new(
+				Opts::new("processed_transactions", "Total number of processed transactions"),
+				&["type"],
+			)
+			.expect("metric is static and thus valid; qed"),
+		}
+	}
+}
+
+impl ExchangeLoopMetrics {
+	/// Update metrics when single block is relayed.
+	pub fn update<P: TransactionProofPipeline>(
+		&self,
+		best_processed_block_number: BlockNumberOf<P>,
+		best_known_block_number: BlockNumberOf<P>,
+		relayed_transactions: RelayedBlockTransactions,
+	) {
+		self.best_block_numbers
+			.with_label_values(&["processed"])
+			.set(best_processed_block_number.into());
+		self.best_block_numbers
+			.with_label_values(&["known"])
+			.set(best_known_block_number.into());
+
+		self.processed_blocks.inc();
+
+		self.processed_transactions
+			.with_label_values(&["total"])
+			.inc_by(relayed_transactions.processed as _);
+		self.processed_transactions
+			.with_label_values(&["relayed"])
+			.inc_by(relayed_transactions.relayed as _);
+		self.processed_transactions
+			.with_label_values(&["failed"])
+			.inc_by(relayed_transactions.failed as _);
+	}
+}

bridges/relays/generic/exchange/src/lib.rs

@@ -0,0 +1,26 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Relaying [`currency-exchange`](../pallet_bridge_currency_exchange/index.html) application
+//! specific data. Currency exchange application allows exchanging tokens between bridged chains.
+//! This module provides entrypoints for crafting and submitting (single and multiple)
+//! proof-of-exchange-at-source-chain transaction(s) to target chain.
+
+#![warn(missing_docs)]
+
+pub mod exchange;
+pub mod exchange_loop;
+pub mod exchange_loop_metrics;

bridges/relays/generic/finality/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "finality-relay"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+description = "Finality proofs relay"
+
+[dependencies]
+async-std = "1.6.5"
+async-trait = "0.1.40"
+backoff = "0.2"
+futures = "0.3.5"
+headers-relay = { path = "../headers" }
+log = "0.4.11"
+num-traits = "0.2"
+relay-utils = { path = "../utils" }
+
+[dev-dependencies]
+parking_lot = "0.11.0"

bridges/relays/generic/finality/src/finality_loop.rs

@@ -0,0 +1,618 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The loop basically reads all missing headers and their finality proofs from the source client.
+//! The proof for the best possible header is then submitted to the target node. The only exception
+//! is the mandatory headers, which we always submit to the target node. For such headers, we
+//! assume that the persistent proof either exists, or will eventually become available.
+
+use crate::{FinalityProof, FinalitySyncPipeline, SourceHeader};
+
+use async_trait::async_trait;
+use backoff::backoff::Backoff;
+use futures::{select, Future, FutureExt, Stream, StreamExt};
+use headers_relay::sync_loop_metrics::SyncLoopMetrics;
+use num_traits::{One, Saturating};
+use relay_utils::{
+	metrics::{start as metrics_start, GlobalMetrics, MetricsParams},
+	relay_loop::Client as RelayClient,
+	retry_backoff, FailedClient, MaybeConnectionError,
+};
+use std::{
+	pin::Pin,
+	time::{Duration, Instant},
+};
+
+/// Finality proof synchronization loop parameters.
+#[derive(Debug, Clone)]
+pub struct FinalitySyncParams {
+	/// Interval at which we check updates on both clients. Normally should be larger than
+	/// `min(source_block_time, target_block_time)`.
+	///
+	/// This parameter may be used to limit transactions rate. Increase the value && you'll get
+	/// infrequent updates => sparse headers => potential slow down of bridge applications, but pallet storage
+	/// won't be super large. Decrease the value to near `source_block_time` and you'll get
+	/// transaction for (almost) every block of the source chain => all source headers will be known
+	/// to the target chain => bridge applications will run faster, but pallet storage may explode
+	/// (but if pruning is there, then it's fine).
+	pub tick: Duration,
+	/// Number of finality proofs to keep in internal buffer between loop wakeups.
+	///
+	/// While in "major syncing" state, we still read finality proofs from the stream. They're stored
+	/// in the internal buffer between loop wakeups. When we're close to the tip of the chain, we may
+	/// meet finality delays if headers are not finalized frequently. So instead of waiting for next
+	/// finality proof to appear in the stream, we may use existing proof from that buffer.
+	pub recent_finality_proofs_limit: usize,
+	/// Timeout before we treat our transactions as lost and restart the whole sync process.
+	pub stall_timeout: Duration,
+}
+
+/// Source client used in finality synchronization loop.
+#[async_trait]
+pub trait SourceClient<P: FinalitySyncPipeline>: RelayClient {
+	/// Stream of new finality proofs. The stream is allowed to miss proofs for some
+	/// headers, even if those headers are mandatory.
+	type FinalityProofsStream: Stream<Item = P::FinalityProof>;
+
+	/// Get best finalized block number.
+	async fn best_finalized_block_number(&self) -> Result<P::Number, Self::Error>;
+
+	/// Get canonical header and its finality proof by number.
+	async fn header_and_finality_proof(
+		&self,
+		number: P::Number,
+	) -> Result<(P::Header, Option<P::FinalityProof>), Self::Error>;
+
+	/// Subscribe to new finality proofs.
+	async fn finality_proofs(&self) -> Result<Self::FinalityProofsStream, Self::Error>;
+}
+
+/// Target client used in finality synchronization loop.
+#[async_trait]
+pub trait TargetClient<P: FinalitySyncPipeline>: RelayClient {
+	/// Get best finalized source block number.
+	async fn best_finalized_source_block_number(&self) -> Result<P::Number, Self::Error>;
+
+	/// Submit header finality proof.
+	async fn submit_finality_proof(&self, header: P::Header, proof: P::FinalityProof) -> Result<(), Self::Error>;
+}
+
+/// Run finality proofs synchronization loop.
+pub fn run<P: FinalitySyncPipeline>(
+	source_client: impl SourceClient<P>,
+	target_client: impl TargetClient<P>,
+	sync_params: FinalitySyncParams,
+	metrics_params: Option<MetricsParams>,
+	exit_signal: impl Future<Output = ()>,
+) {
+	let exit_signal = exit_signal.shared();
+
+	let metrics_global = GlobalMetrics::default();
+	let metrics_sync = SyncLoopMetrics::default();
+	let metrics_enabled = metrics_params.is_some();
+	metrics_start(
+		format!("{}_to_{}_Sync", P::SOURCE_NAME, P::TARGET_NAME),
+		metrics_params,
+		&metrics_global,
+		&metrics_sync,
+	);
+
+	relay_utils::relay_loop::run(
+		relay_utils::relay_loop::RECONNECT_DELAY,
+		source_client,
+		target_client,
+		|source_client, target_client| {
+			run_until_connection_lost(
+				source_client,
+				target_client,
+				sync_params.clone(),
+				if metrics_enabled {
+					Some(metrics_global.clone())
+				} else {
+					None
+				},
+				if metrics_enabled {
+					Some(metrics_sync.clone())
+				} else {
+					None
+				},
+				exit_signal.clone(),
+			)
+		},
+	);
+}
+
+/// Unjustified headers container. Ordered by header number.
+pub(crate) type UnjustifiedHeaders<H> = Vec<H>;
+/// Finality proofs container. Ordered by target header number.
+pub(crate) type FinalityProofs<P> = Vec<(
+	<P as FinalitySyncPipeline>::Number,
+	<P as FinalitySyncPipeline>::FinalityProof,
+)>;
+/// Reference to finality proofs container.
+pub(crate) type FinalityProofsRef<'a, P> = &'a [(
+	<P as FinalitySyncPipeline>::Number,
+	<P as FinalitySyncPipeline>::FinalityProof,
+)];
+
+/// Error that may happen inside finality synchronization loop.
+#[derive(Debug)]
+pub(crate) enum Error<P: FinalitySyncPipeline, SourceError, TargetError> {
+	/// Source client request has failed with given error.
+	Source(SourceError),
+	/// Target client request has failed with given error.
+	Target(TargetError),
+	/// Finality proof for mandatory header is missing from the source node.
+	MissingMandatoryFinalityProof(P::Number),
+	/// The synchronization has stalled.
+	Stalled,
+}
+
+impl<P, SourceError, TargetError> Error<P, SourceError, TargetError>
+where
+	P: FinalitySyncPipeline,
+	SourceError: MaybeConnectionError,
+	TargetError: MaybeConnectionError,
+{
+	fn fail_if_connection_error(&self) -> Result<(), FailedClient> {
+		match *self {
+			Error::Source(ref error) if error.is_connection_error() => Err(FailedClient::Source),
+			Error::Target(ref error) if error.is_connection_error() => Err(FailedClient::Target),
+			Error::Stalled => Err(FailedClient::Both),
+			_ => Ok(()),
+		}
+	}
+}
+
+/// Information about transaction that we have submitted.
+#[derive(Debug, Clone)]
+struct Transaction<Number> {
+	/// Time when we have submitted this transaction.
+	pub time: Instant,
+	/// The number of the header we have submitted.
+	pub submitted_header_number: Number,
+}
+
+/// Finality proofs stream that may be restarted.
+pub(crate) struct RestartableFinalityProofsStream<S> {
+	/// Flag that the stream needs to be restarted.
+	pub(crate) needs_restart: bool,
+	/// The stream itself.
+	stream: Pin<Box<S>>,
+}
+
+#[cfg(test)]
+impl<S> From<S> for RestartableFinalityProofsStream<S> {
+	fn from(stream: S) -> Self {
+		RestartableFinalityProofsStream {
+			needs_restart: false,
+			stream: Box::pin(stream),
+		}
+	}
+}
+
+/// Finality synchronization loop state.
+struct FinalityLoopState<'a, P: FinalitySyncPipeline, FinalityProofsStream> {
+	/// Synchronization loop progress.
+	progress: &'a mut (Instant, Option<P::Number>),
+	/// Finality proofs stream.
+	finality_proofs_stream: &'a mut RestartableFinalityProofsStream<FinalityProofsStream>,
+	/// Recent finality proofs that we have read from the stream.
+	recent_finality_proofs: &'a mut FinalityProofs<P>,
+	/// Last transaction that we have submitted to the target node.
+	last_transaction: Option<Transaction<P::Number>>,
+}
+
+async fn run_until_connection_lost<P: FinalitySyncPipeline>(
+	source_client: impl SourceClient<P>,
+	target_client: impl TargetClient<P>,
+	sync_params: FinalitySyncParams,
+	metrics_global: Option<GlobalMetrics>,
+	metrics_sync: Option<SyncLoopMetrics>,
+	exit_signal: impl Future<Output = ()>,
+) -> Result<(), FailedClient> {
+	let restart_finality_proofs_stream = || async {
+		source_client.finality_proofs().await.map_err(|error| {
+			log::error!(
+				target: "bridge",
+				"Failed to subscribe to {} justifications: {:?}. Going to reconnect",
+				P::SOURCE_NAME,
+				error,
+			);
+
+			FailedClient::Source
+		})
+	};
+
+	let exit_signal = exit_signal.fuse();
+	futures::pin_mut!(exit_signal);
+
+	let mut finality_proofs_stream = RestartableFinalityProofsStream {
+		needs_restart: false,
+		stream: Box::pin(restart_finality_proofs_stream().await?),
+	};
+	let mut recent_finality_proofs = Vec::new();
+
+	let mut progress = (Instant::now(), None);
+	let mut retry_backoff = retry_backoff();
+	let mut last_transaction = None;
+
+	loop {
+		// run loop iteration
+		let iteration_result = run_loop_iteration(
+			&source_client,
+			&target_client,
+			FinalityLoopState {
+				progress: &mut progress,
+				finality_proofs_stream: &mut finality_proofs_stream,
+				recent_finality_proofs: &mut recent_finality_proofs,
+				last_transaction: last_transaction.clone(),
+			},
+			&sync_params,
+			&metrics_sync,
+		)
+		.await;
+
+		// update global metrics
+		if let Some(ref metrics_global) = metrics_global {
+			metrics_global.update().await;
+		}
+
+		// deal with errors
+		let next_tick = match iteration_result {
+			Ok(updated_last_transaction) => {
+				last_transaction = updated_last_transaction;
+				retry_backoff.reset();
+				sync_params.tick
+			}
+			Err(error) => {
+				log::error!(target: "bridge", "Finality sync loop iteration has failed with error: {:?}", error);
+				error.fail_if_connection_error()?;
+				retry_backoff
+					.next_backoff()
+					.unwrap_or(relay_utils::relay_loop::RECONNECT_DELAY)
+			}
+		};
+		if finality_proofs_stream.needs_restart {
+			log::warn!(target: "bridge", "{} finality proofs stream is being restarted", P::SOURCE_NAME);
+
+			finality_proofs_stream.needs_restart = false;
+			finality_proofs_stream.stream = Box::pin(restart_finality_proofs_stream().await?);
+		}
+
+		// wait till exit signal, or new source block
+		select! {
+			_ = async_std::task::sleep(next_tick).fuse() => {},
+			_ = exit_signal => return Ok(()),
+		}
+	}
+}
+
+async fn run_loop_iteration<P, SC, TC>(
+	source_client: &SC,
+	target_client: &TC,
+	state: FinalityLoopState<'_, P, SC::FinalityProofsStream>,
+	sync_params: &FinalitySyncParams,
+	metrics_sync: &Option<SyncLoopMetrics>,
+) -> Result<Option<Transaction<P::Number>>, Error<P, SC::Error, TC::Error>>
+where
+	P: FinalitySyncPipeline,
+	SC: SourceClient<P>,
+	TC: TargetClient<P>,
+{
+	// read best source headers ids from source and target nodes
+	let best_number_at_source = source_client
+		.best_finalized_block_number()
+		.await
+		.map_err(Error::Source)?;
+	let best_number_at_target = target_client
+		.best_finalized_source_block_number()
+		.await
+		.map_err(Error::Target)?;
+	if let Some(ref metrics_sync) = *metrics_sync {
+		metrics_sync.update_best_block_at_source(best_number_at_source);
+		metrics_sync.update_best_block_at_target(best_number_at_target);
+	}
+	*state.progress = print_sync_progress::<P>(*state.progress, best_number_at_source, best_number_at_target);
+
+	// if we have already submitted header, then we just need to wait for it
+	// if we're waiting too much, then we believe our transaction has been lost and restart sync
+	if let Some(last_transaction) = state.last_transaction {
+		if best_number_at_target >= last_transaction.submitted_header_number {
+			// transaction has been mined && we can continue
+		} else if last_transaction.time.elapsed() > sync_params.stall_timeout {
+			log::error!(
+				target: "bridge",
+				"Finality synchronization from {} to {} has stalled. Going to restart",
+				P::SOURCE_NAME,
+				P::TARGET_NAME,
+			);
+
+			return Err(Error::Stalled);
+		} else {
+			return Ok(Some(last_transaction));
+		}
+	}
+
+	// submit new header if we have something new
+	match select_header_to_submit(
+		source_client,
+		target_client,
+		state.finality_proofs_stream,
+		state.recent_finality_proofs,
+		best_number_at_source,
+		best_number_at_target,
+		sync_params,
+	)
+	.await?
+	{
+		Some((header, justification)) => {
+			let new_transaction = Transaction {
+				time: Instant::now(),
+				submitted_header_number: header.number(),
+			};
+
+			log::debug!(
+				target: "bridge",
+				"Going to submit finality proof of {} header #{:?} to {}",
+				P::SOURCE_NAME,
+				new_transaction.submitted_header_number,
+				P::TARGET_NAME,
+			);
+
+			target_client
+				.submit_finality_proof(header, justification)
+				.await
+				.map_err(Error::Target)?;
+			Ok(Some(new_transaction))
+		}
+		None => Ok(None),
+	}
+}
+
+async fn select_header_to_submit<P, SC, TC>(
+	source_client: &SC,
+	target_client: &TC,
+	finality_proofs_stream: &mut RestartableFinalityProofsStream<SC::FinalityProofsStream>,
+	recent_finality_proofs: &mut FinalityProofs<P>,
+	best_number_at_source: P::Number,
+	best_number_at_target: P::Number,
+	sync_params: &FinalitySyncParams,
+) -> Result<Option<(P::Header, P::FinalityProof)>, Error<P, SC::Error, TC::Error>>
+where
+	P: FinalitySyncPipeline,
+	SC: SourceClient<P>,
+	TC: TargetClient<P>,
+{
+	// to see that the loop is progressing
+	log::trace!(
+		target: "bridge",
+		"Considering range of headers ({:?}; {:?}]",
+		best_number_at_target,
+		best_number_at_source,
+	);
+
+	// read missing headers. if we see that the header schedules GRANDPA change, we need to
+	// submit this header
+	let selected_finality_proof = read_missing_headers::<P, SC, TC>(
+		source_client,
+		target_client,
+		best_number_at_source,
+		best_number_at_target,
+	)
+	.await?;
+	let (mut unjustified_headers, mut selected_finality_proof) = match selected_finality_proof {
+		SelectedFinalityProof::Mandatory(header, finality_proof) => return Ok(Some((header, finality_proof))),
+		SelectedFinalityProof::Regular(unjustified_headers, header, finality_proof) => {
+			(unjustified_headers, Some((header, finality_proof)))
+		}
+		SelectedFinalityProof::None(unjustified_headers) => (unjustified_headers, None),
+	};
+
+	// all headers that are missing from the target client are non-mandatory
+	// => even if we have already selected some header and its persistent finality proof,
+	// we may try to select better header by reading non-persistent proofs from the stream
+	read_finality_proofs_from_stream::<P, _>(finality_proofs_stream, recent_finality_proofs);
+	selected_finality_proof = select_better_recent_finality_proof::<P>(
+		recent_finality_proofs,
+		&mut unjustified_headers,
+		selected_finality_proof,
+	);
+
+	// remove obsolete 'recent' finality proofs + keep its size under certain limit
+	let oldest_finality_proof_to_keep = selected_finality_proof
+		.as_ref()
+		.map(|(header, _)| header.number())
+		.unwrap_or(best_number_at_target);
+	prune_recent_finality_proofs::<P>(
+		oldest_finality_proof_to_keep,
+		recent_finality_proofs,
+		sync_params.recent_finality_proofs_limit,
+	);
+
+	Ok(selected_finality_proof)
+}
+
+/// Finality proof that has been selected by the `read_missing_headers` function.
+pub(crate) enum SelectedFinalityProof<Header, FinalityProof> {
+	/// Mandatory header and its proof has been selected. We shall submit proof for this header.
+	Mandatory(Header, FinalityProof),
+	/// Regular header and its proof has been selected. We may submit this proof, or proof for
+	/// some better header.
+	Regular(UnjustifiedHeaders<Header>, Header, FinalityProof),
+	/// We haven't found any missing header with persistent proof at the target client.
+	None(UnjustifiedHeaders<Header>),
+}
+
+/// Read missing headers and their persistent finality proofs from the target client.
+///
+/// If we have found some header with known proof, it is returned.
+/// Otherwise, `SelectedFinalityProof::None` is returned.
+///
+/// Unless we have found mandatory header, all missing headers are collected and returned.
+pub(crate) async fn read_missing_headers<P: FinalitySyncPipeline, SC: SourceClient<P>, TC: TargetClient<P>>(
+	source_client: &SC,
+	_target_client: &TC,
+	best_number_at_source: P::Number,
+	best_number_at_target: P::Number,
+) -> Result<SelectedFinalityProof<P::Header, P::FinalityProof>, Error<P, SC::Error, TC::Error>> {
+	let mut unjustified_headers = Vec::new();
+	let mut selected_finality_proof = None;
+	let mut header_number = best_number_at_target + One::one();
+	while header_number <= best_number_at_source {
+		let (header, finality_proof) = source_client
+			.header_and_finality_proof(header_number)
+			.await
+			.map_err(Error::Source)?;
+		let is_mandatory = header.is_mandatory();
+
+		match (is_mandatory, finality_proof) {
+			(true, Some(finality_proof)) => {
+				log::trace!(target: "bridge", "Header {:?} is mandatory", header_number);
+				return Ok(SelectedFinalityProof::Mandatory(header, finality_proof));
+			}
+			(true, None) => return Err(Error::MissingMandatoryFinalityProof(header.number())),
+			(false, Some(finality_proof)) => {
+				log::trace!(target: "bridge", "Header {:?} has persistent finality proof", header_number);
+				unjustified_headers.clear();
+				selected_finality_proof = Some((header, finality_proof));
+			}
+			(false, None) => {
+				unjustified_headers.push(header);
+			}
+		}
+
+		header_number = header_number + One::one();
+	}
+
+	Ok(match selected_finality_proof {
+		Some((header, proof)) => SelectedFinalityProof::Regular(unjustified_headers, header, proof),
+		None => SelectedFinalityProof::None(unjustified_headers),
+	})
+}
+
+/// Read finality proofs from the stream.
+pub(crate) fn read_finality_proofs_from_stream<P: FinalitySyncPipeline, FPS: Stream<Item = P::FinalityProof>>(
+	finality_proofs_stream: &mut RestartableFinalityProofsStream<FPS>,
+	recent_finality_proofs: &mut FinalityProofs<P>,
+) {
+	loop {
+		let next_proof = finality_proofs_stream.stream.next();
+		let finality_proof = match next_proof.now_or_never() {
+			Some(Some(finality_proof)) => finality_proof,
+			Some(None) => {
+				finality_proofs_stream.needs_restart = true;
+				break;
+			}
+			None => break,
+		};
+
+		recent_finality_proofs.push((finality_proof.target_header_number(), finality_proof));
+	}
+}
+
+/// Try to select better header and its proof, given finality proofs that we
+/// have recently read from the stream.
+pub(crate) fn select_better_recent_finality_proof<P: FinalitySyncPipeline>(
+	recent_finality_proofs: FinalityProofsRef<P>,
+	unjustified_headers: &mut UnjustifiedHeaders<P::Header>,
+	selected_finality_proof: Option<(P::Header, P::FinalityProof)>,
+) -> Option<(P::Header, P::FinalityProof)> {
+	if unjustified_headers.is_empty() || recent_finality_proofs.is_empty() {
+		return selected_finality_proof;
+	}
+
+	const NOT_EMPTY_PROOF: &str = "we have checked that the vec is not empty; qed";
+
+	// we need proofs for headers in range unjustified_range_begin..=unjustified_range_end
+	let unjustified_range_begin = unjustified_headers.first().expect(NOT_EMPTY_PROOF).number();
+	let unjustified_range_end = unjustified_headers.last().expect(NOT_EMPTY_PROOF).number();
+
+	// we have proofs for headers in range buffered_range_begin..=buffered_range_end
+	let buffered_range_begin = recent_finality_proofs.first().expect(NOT_EMPTY_PROOF).0;
+	let buffered_range_end = recent_finality_proofs.last().expect(NOT_EMPTY_PROOF).0;
+
+	// we have two ranges => find intersection
+	let intersection_begin = std::cmp::max(unjustified_range_begin, buffered_range_begin);
+	let intersection_end = std::cmp::min(unjustified_range_end, buffered_range_end);
+	let intersection = intersection_begin..=intersection_end;
+
+	// find last proof from intersection
+	let selected_finality_proof_index = recent_finality_proofs
+		.binary_search_by_key(intersection.end(), |(number, _)| *number)
+		.unwrap_or_else(|index| index.saturating_sub(1));
+	let (selected_header_number, finality_proof) = &recent_finality_proofs[selected_finality_proof_index];
+	if !intersection.contains(selected_header_number) {
+		return selected_finality_proof;
+	}
+
+	// now remove all obsolete headers and extract selected header
+	let selected_header_position = unjustified_headers
+		.binary_search_by_key(selected_header_number, |header| header.number())
+		.expect("unjustified_headers contain all headers from intersection; qed");
+	let selected_header = unjustified_headers.swap_remove(selected_header_position);
+	Some((selected_header, finality_proof.clone()))
+}
+
+pub(crate) fn prune_recent_finality_proofs<P: FinalitySyncPipeline>(
+	justified_header_number: P::Number,
+	recent_finality_proofs: &mut FinalityProofs<P>,
+	recent_finality_proofs_limit: usize,
+) {
+	let position =
+		recent_finality_proofs.binary_search_by_key(&justified_header_number, |(header_number, _)| *header_number);
+
+	// remove all obsolete elements
+	*recent_finality_proofs = recent_finality_proofs.split_off(
+		position
+			.map(|position| position + 1)
+			.unwrap_or_else(|position| position),
+	);
+
+	// now - limit vec by size
+	let split_index = recent_finality_proofs
+		.len()
+		.saturating_sub(recent_finality_proofs_limit);
+	*recent_finality_proofs = recent_finality_proofs.split_off(split_index);
+}
+
+fn print_sync_progress<P: FinalitySyncPipeline>(
+	progress_context: (Instant, Option<P::Number>),
+	best_number_at_source: P::Number,
+	best_number_at_target: P::Number,
+) -> (Instant, Option<P::Number>) {
+	let (prev_time, prev_best_number_at_target) = progress_context;
+	let now = Instant::now();
+
+	let need_update = now - prev_time > Duration::from_secs(10)
+		|| prev_best_number_at_target
+			.map(|prev_best_number_at_target| {
+				best_number_at_target.saturating_sub(prev_best_number_at_target) > 10.into()
+			})
+			.unwrap_or(true);
+
+	if !need_update {
+		return (prev_time, prev_best_number_at_target);
+	}
+
+	log::info!(
+		target: "bridge",
+		"Synced {:?} of {:?} headers",
+		best_number_at_target,
+		best_number_at_source,
+	);
+	(now, Some(best_number_at_target))
+}

bridges/relays/generic/finality/src/finality_loop_tests.rs

@@ -0,0 +1,404 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Tests for finality synchronization loop.
+
+#![cfg(test)]
+
+use crate::finality_loop::{
+	prune_recent_finality_proofs, read_finality_proofs_from_stream, run, select_better_recent_finality_proof,
+	FinalityProofs, FinalitySyncParams, SourceClient, TargetClient,
+};
+use crate::{FinalityProof, FinalitySyncPipeline, SourceHeader};
+
+use async_trait::async_trait;
+use futures::{FutureExt, Stream, StreamExt};
+use parking_lot::Mutex;
+use relay_utils::{relay_loop::Client as RelayClient, MaybeConnectionError};
+use std::{collections::HashMap, pin::Pin, sync::Arc, time::Duration};
+
+type IsMandatory = bool;
+type TestNumber = u64;
+
+#[derive(Debug, Clone)]
+enum TestError {
+	NonConnection,
+}
+
+impl MaybeConnectionError for TestError {
+	fn is_connection_error(&self) -> bool {
+		false
+	}
+}
+
+#[derive(Debug, Clone)]
+struct TestFinalitySyncPipeline;
+
+impl FinalitySyncPipeline for TestFinalitySyncPipeline {
+	const SOURCE_NAME: &'static str = "TestSource";
+	const TARGET_NAME: &'static str = "TestTarget";
+
+	type Hash = u64;
+	type Number = TestNumber;
+	type Header = TestSourceHeader;
+	type FinalityProof = TestFinalityProof;
+}
+
+#[derive(Debug, Clone, PartialEq)]
+struct TestSourceHeader(IsMandatory, TestNumber);
+
+impl SourceHeader<TestNumber> for TestSourceHeader {
+	fn number(&self) -> TestNumber {
+		self.1
+	}
+
+	fn is_mandatory(&self) -> bool {
+		self.0
+	}
+}
+
+#[derive(Debug, Clone, PartialEq)]
+struct TestFinalityProof(TestNumber);
+
+impl FinalityProof<TestNumber> for TestFinalityProof {
+	fn target_header_number(&self) -> TestNumber {
+		self.0
+	}
+}
+
+#[derive(Debug, Clone, Default)]
+struct ClientsData {
+	source_best_block_number: TestNumber,
+	source_headers: HashMap<TestNumber, (TestSourceHeader, Option<TestFinalityProof>)>,
+	source_proofs: Vec<TestFinalityProof>,
+
+	target_best_block_number: TestNumber,
+	target_headers: Vec<(TestSourceHeader, TestFinalityProof)>,
+}
+
+#[derive(Clone)]
+struct TestSourceClient {
+	on_method_call: Arc<dyn Fn(&mut ClientsData) + Send + Sync>,
+	data: Arc<Mutex<ClientsData>>,
+}
+
+#[async_trait]
+impl RelayClient for TestSourceClient {
+	type Error = TestError;
+
+	async fn reconnect(&mut self) -> Result<(), TestError> {
+		unreachable!()
+	}
+}
+
+#[async_trait]
+impl SourceClient<TestFinalitySyncPipeline> for TestSourceClient {
+	type FinalityProofsStream = Pin<Box<dyn Stream<Item = TestFinalityProof>>>;
+
+	async fn best_finalized_block_number(&self) -> Result<TestNumber, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(&mut *data);
+		Ok(data.source_best_block_number)
+	}
+
+	async fn header_and_finality_proof(
+		&self,
+		number: TestNumber,
+	) -> Result<(TestSourceHeader, Option<TestFinalityProof>), TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(&mut *data);
+		data.source_headers
+			.get(&number)
+			.cloned()
+			.ok_or(TestError::NonConnection)
+	}
+
+	async fn finality_proofs(&self) -> Result<Self::FinalityProofsStream, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(&mut *data);
+		Ok(futures::stream::iter(data.source_proofs.clone()).boxed())
+	}
+}
+
+#[derive(Clone)]
+struct TestTargetClient {
+	on_method_call: Arc<dyn Fn(&mut ClientsData) + Send + Sync>,
+	data: Arc<Mutex<ClientsData>>,
+}
+
+#[async_trait]
+impl RelayClient for TestTargetClient {
+	type Error = TestError;
+
+	async fn reconnect(&mut self) -> Result<(), TestError> {
+		unreachable!()
+	}
+}
+
+#[async_trait]
+impl TargetClient<TestFinalitySyncPipeline> for TestTargetClient {
+	async fn best_finalized_source_block_number(&self) -> Result<TestNumber, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(&mut *data);
+		Ok(data.target_best_block_number)
+	}
+
+	async fn submit_finality_proof(&self, header: TestSourceHeader, proof: TestFinalityProof) -> Result<(), TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(&mut *data);
+		data.target_best_block_number = header.number();
+		data.target_headers.push((header, proof));
+		Ok(())
+	}
+}
+
+fn run_sync_loop(state_function: impl Fn(&mut ClientsData) -> bool + Send + Sync + 'static) -> ClientsData {
+	let (exit_sender, exit_receiver) = futures::channel::mpsc::unbounded();
+	let internal_state_function: Arc<dyn Fn(&mut ClientsData) + Send + Sync> = Arc::new(move |data| {
+		if state_function(data) {
+			exit_sender.unbounded_send(()).unwrap();
+		}
+	});
+	let clients_data = Arc::new(Mutex::new(ClientsData {
+		source_best_block_number: 10,
+		source_headers: vec![
+			(6, (TestSourceHeader(false, 6), None)),
+			(7, (TestSourceHeader(false, 7), Some(TestFinalityProof(7)))),
+			(8, (TestSourceHeader(true, 8), Some(TestFinalityProof(8)))),
+			(9, (TestSourceHeader(false, 9), Some(TestFinalityProof(9)))),
+			(10, (TestSourceHeader(false, 10), None)),
+		]
+		.into_iter()
+		.collect(),
+		source_proofs: vec![TestFinalityProof(12), TestFinalityProof(14)],
+
+		target_best_block_number: 5,
+		target_headers: vec![],
+	}));
+	let source_client = TestSourceClient {
+		on_method_call: internal_state_function.clone(),
+		data: clients_data.clone(),
+	};
+	let target_client = TestTargetClient {
+		on_method_call: internal_state_function,
+		data: clients_data.clone(),
+	};
+	let sync_params = FinalitySyncParams {
+		tick: Duration::from_secs(0),
+		recent_finality_proofs_limit: 1024,
+		stall_timeout: Duration::from_secs(1),
+	};
+
+	run(
+		source_client,
+		target_client,
+		sync_params,
+		None,
+		exit_receiver.into_future().map(|(_, _)| ()),
+	);
+
+	let clients_data = clients_data.lock().clone();
+	clients_data
+}
+
+#[test]
+fn finality_sync_loop_works() {
+	let client_data = run_sync_loop(|data| {
+		// header#7 has persistent finality proof, but it isn't mandatory => it isn't submitted, because
+		// header#8 has persistent finality proof && it is mandatory => it is submitted
+		// header#9 has persistent finality proof, but it isn't mandatory => it is submitted, because
+		//   there are no more persistent finality proofs
+		//
+		// once this ^^^ is done, we generate more blocks && read proof for blocks 12 and 14 from the stream
+		if data.target_best_block_number == 9 {
+			data.source_best_block_number = 14;
+			data.source_headers.insert(11, (TestSourceHeader(false, 11), None));
+			data.source_headers
+				.insert(12, (TestSourceHeader(false, 12), Some(TestFinalityProof(12))));
+			data.source_headers.insert(13, (TestSourceHeader(false, 13), None));
+			data.source_headers
+				.insert(14, (TestSourceHeader(false, 14), Some(TestFinalityProof(14))));
+		}
+		// once this ^^^ is done, we generate more blocks && read persistent proof for block 16
+		if data.target_best_block_number == 14 {
+			data.source_best_block_number = 17;
+			data.source_headers.insert(15, (TestSourceHeader(false, 15), None));
+			data.source_headers
+				.insert(16, (TestSourceHeader(false, 16), Some(TestFinalityProof(16))));
+			data.source_headers.insert(17, (TestSourceHeader(false, 17), None));
+		}
+
+		data.target_best_block_number == 16
+	});
+
+	assert_eq!(
+		client_data.target_headers,
+		vec![
+			// before adding 11..14: finality proof for mandatory header#8
+			(TestSourceHeader(true, 8), TestFinalityProof(8)),
+			// before adding 11..14: persistent finality proof for non-mandatory header#9
+			(TestSourceHeader(false, 9), TestFinalityProof(9)),
+			// after adding 11..14: ephemeral finality proof for non-mandatory header#14
+			(TestSourceHeader(false, 14), TestFinalityProof(14)),
+			// after adding 15..17: persistent finality proof for non-mandatory header#16
+			(TestSourceHeader(false, 16), TestFinalityProof(16)),
+		],
+	);
+}
+
+#[test]
+fn select_better_recent_finality_proof_works() {
+	// if there are no unjustified headers, nothing is changed
+	assert_eq!(
+		select_better_recent_finality_proof::<TestFinalitySyncPipeline>(
+			&[(5, TestFinalityProof(5))],
+			&mut vec![],
+			Some((TestSourceHeader(false, 2), TestFinalityProof(2))),
+		),
+		Some((TestSourceHeader(false, 2), TestFinalityProof(2))),
+	);
+
+	// if there are no recent finality proofs, nothing is changed
+	assert_eq!(
+		select_better_recent_finality_proof::<TestFinalitySyncPipeline>(
+			&[],
+			&mut vec![TestSourceHeader(false, 5)],
+			Some((TestSourceHeader(false, 2), TestFinalityProof(2))),
+		),
+		Some((TestSourceHeader(false, 2), TestFinalityProof(2))),
+	);
+
+	// if there's no intersection between recent finality proofs and unjustified headers, nothing is changed
+	let mut unjustified_headers = vec![TestSourceHeader(false, 9), TestSourceHeader(false, 10)];
+	assert_eq!(
+		select_better_recent_finality_proof::<TestFinalitySyncPipeline>(
+			&[(1, TestFinalityProof(1)), (4, TestFinalityProof(4))],
+			&mut unjustified_headers,
+			Some((TestSourceHeader(false, 2), TestFinalityProof(2))),
+		),
+		Some((TestSourceHeader(false, 2), TestFinalityProof(2))),
+	);
+
+	// if there's intersection between recent finality proofs and unjustified headers, but there are no
+	// proofs in this intersection, nothing is changed
+	let mut unjustified_headers = vec![
+		TestSourceHeader(false, 8),
+		TestSourceHeader(false, 9),
+		TestSourceHeader(false, 10),
+	];
+	assert_eq!(
+		select_better_recent_finality_proof::<TestFinalitySyncPipeline>(
+			&[(7, TestFinalityProof(7)), (11, TestFinalityProof(11))],
+			&mut unjustified_headers,
+			Some((TestSourceHeader(false, 2), TestFinalityProof(2))),
+		),
+		Some((TestSourceHeader(false, 2), TestFinalityProof(2))),
+	);
+	assert_eq!(
+		unjustified_headers,
+		vec![
+			TestSourceHeader(false, 8),
+			TestSourceHeader(false, 9),
+			TestSourceHeader(false, 10)
+		]
+	);
+
+	// if there's intersection between recent finality proofs and unjustified headers and there's
+	// a proof in this intersection:
+	// - this better (last from intersection) proof is selected;
+	// - 'obsolete' unjustified headers are pruned.
+	let mut unjustified_headers = vec![
+		TestSourceHeader(false, 8),
+		TestSourceHeader(false, 9),
+		TestSourceHeader(false, 10),
+	];
+	assert_eq!(
+		select_better_recent_finality_proof::<TestFinalitySyncPipeline>(
+			&[(7, TestFinalityProof(7)), (9, TestFinalityProof(9))],
+			&mut unjustified_headers,
+			Some((TestSourceHeader(false, 2), TestFinalityProof(2))),
+		),
+		Some((TestSourceHeader(false, 9), TestFinalityProof(9))),
+	);
+}
+
+#[test]
+fn read_finality_proofs_from_stream_works() {
+	// when stream is currently empty, nothing is changed
+	let mut recent_finality_proofs = vec![(1, TestFinalityProof(1))];
+	let mut stream = futures::stream::pending().into();
+	read_finality_proofs_from_stream::<TestFinalitySyncPipeline, _>(&mut stream, &mut recent_finality_proofs);
+	assert_eq!(recent_finality_proofs, vec![(1, TestFinalityProof(1))]);
+	assert_eq!(stream.needs_restart, false);
+
+	// when stream has entry with target, it is added to the recent proofs container
+	let mut stream = futures::stream::iter(vec![TestFinalityProof(4)])
+		.chain(futures::stream::pending())
+		.into();
+	read_finality_proofs_from_stream::<TestFinalitySyncPipeline, _>(&mut stream, &mut recent_finality_proofs);
+	assert_eq!(
+		recent_finality_proofs,
+		vec![(1, TestFinalityProof(1)), (4, TestFinalityProof(4))]
+	);
+	assert_eq!(stream.needs_restart, false);
+
+	// when stream has ended, we'll need to restart it
+	let mut stream = futures::stream::empty().into();
+	read_finality_proofs_from_stream::<TestFinalitySyncPipeline, _>(&mut stream, &mut recent_finality_proofs);
+	assert_eq!(
+		recent_finality_proofs,
+		vec![(1, TestFinalityProof(1)), (4, TestFinalityProof(4))]
+	);
+	assert_eq!(stream.needs_restart, true);
+}
+
+#[test]
+fn prune_recent_finality_proofs_works() {
+	let original_recent_finality_proofs: FinalityProofs<TestFinalitySyncPipeline> = vec![
+		(10, TestFinalityProof(10)),
+		(13, TestFinalityProof(13)),
+		(15, TestFinalityProof(15)),
+		(17, TestFinalityProof(17)),
+		(19, TestFinalityProof(19)),
+	]
+	.into_iter()
+	.collect();
+
+	// when there's proof for justified header in the vec
+	let mut recent_finality_proofs = original_recent_finality_proofs.clone();
+	prune_recent_finality_proofs::<TestFinalitySyncPipeline>(10, &mut recent_finality_proofs, 1024);
+	assert_eq!(&original_recent_finality_proofs[1..], recent_finality_proofs,);
+
+	// when there are no proof for justified header in the vec
+	let mut recent_finality_proofs = original_recent_finality_proofs.clone();
+	prune_recent_finality_proofs::<TestFinalitySyncPipeline>(11, &mut recent_finality_proofs, 1024);
+	assert_eq!(&original_recent_finality_proofs[1..], recent_finality_proofs,);
+
+	// when there are too many entries after initial prune && they also need to be pruned
+	let mut recent_finality_proofs = original_recent_finality_proofs.clone();
+	prune_recent_finality_proofs::<TestFinalitySyncPipeline>(10, &mut recent_finality_proofs, 2);
+	assert_eq!(&original_recent_finality_proofs[3..], recent_finality_proofs,);
+
+	// when last entry is pruned
+	let mut recent_finality_proofs = original_recent_finality_proofs.clone();
+	prune_recent_finality_proofs::<TestFinalitySyncPipeline>(19, &mut recent_finality_proofs, 2);
+	assert_eq!(&original_recent_finality_proofs[5..], recent_finality_proofs,);
+
+	// when post-last entry is pruned
+	let mut recent_finality_proofs = original_recent_finality_proofs.clone();
+	prune_recent_finality_proofs::<TestFinalitySyncPipeline>(20, &mut recent_finality_proofs, 2);
+	assert_eq!(&original_recent_finality_proofs[5..], recent_finality_proofs,);
+}

bridges/relays/generic/finality/src/lib.rs

@@ -0,0 +1,58 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! This crate has single entrypoint to run synchronization loop that is built around finality
+//! proofs, as opposed to headers synchronization loop, which is built around headers. The headers
+//! are still submitted to the target node, but are treated as auxiliary data as we are not trying
+//! to submit all source headers to the target node.
+
+pub use crate::finality_loop::{run, FinalitySyncParams, SourceClient, TargetClient};
+
+use std::fmt::Debug;
+
+mod finality_loop;
+mod finality_loop_tests;
+
+/// Finality proofs synchronization pipeline.
+pub trait FinalitySyncPipeline: Clone + Debug + Send + Sync {
+	/// Name of the finality proofs source.
+	const SOURCE_NAME: &'static str;
+	/// Name of the finality proofs target.
+	const TARGET_NAME: &'static str;
+
+	/// Headers we're syncing are identified by this hash.
+	type Hash: Eq + Clone + Copy + Send + Sync + Debug;
+	/// Headers we're syncing are identified by this number.
+	type Number: relay_utils::BlockNumberBase;
+	/// Type of header that we're syncing.
+	type Header: SourceHeader<Self::Number>;
+	/// Finality proof type.
+	type FinalityProof: FinalityProof<Self::Number>;
+}
+
+/// Header that we're receiving from source node.
+pub trait SourceHeader<Number>: Clone + Debug + PartialEq + Send + Sync {
+	/// Returns number of header.
+	fn number(&self) -> Number;
+	/// Returns true if this header needs to be submitted to target node.
+	fn is_mandatory(&self) -> bool;
+}
+
+/// Abstract finality proof that is justifying block finality.
+pub trait FinalityProof<Number>: Clone + Send + Sync + Debug {
+	/// Return number of header that this proof is generated for.
+	fn target_header_number(&self) -> Number;
+}

bridges/relays/generic/headers/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "headers-relay"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+
+[dependencies]
+async-std = "1.6.5"
+async-trait = "0.1.40"
+backoff = "0.2"
+futures = "0.3.5"
+linked-hash-map = "0.5.3"
+log = "0.4.11"
+num-traits = "0.2"
+parking_lot = "0.11.0"
+relay-utils = { path = "../utils" }

bridges/relays/generic/headers/src/lib.rs

@@ -0,0 +1,33 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Relaying source chain headers to target chain. This module provides entrypoint
+//! that starts reading new headers from source chain and submit these headers as
+//! module/contract transactions to the target chain. Module/contract on the target
+//! chain is a light-client of the source chain. All other trustless bridge
+//! applications are built using this light-client, so running headers-relay is
+//! essential for running all other bridge applications.
+
+// required for futures::select!
+#![recursion_limit = "1024"]
+#![warn(missing_docs)]
+
+pub mod headers;
+pub mod sync;
+pub mod sync_loop;
+pub mod sync_loop_metrics;
+pub mod sync_loop_tests;
+pub mod sync_types;

bridges/relays/generic/headers/src/sync.rs

@@ -0,0 +1,523 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Headers synchronization context. This structure wraps headers queue and is
+//! able to choose: which headers to read from the source chain? Which headers
+//! to submit to the target chain? The context makes decisions basing on parameters
+//! passed using `HeadersSyncParams` structure.
+
+use crate::headers::QueuedHeaders;
+use crate::sync_types::{HeaderIdOf, HeaderStatus, HeadersSyncPipeline, QueuedHeader};
+use num_traits::{One, Saturating, Zero};
+
+/// Common sync params.
+#[derive(Debug, Clone)]
+pub struct HeadersSyncParams {
+	/// Maximal number of ethereum headers to pre-download.
+	pub max_future_headers_to_download: usize,
+	/// Maximal number of active (we believe) submit header transactions.
+	pub max_headers_in_submitted_status: usize,
+	/// Maximal number of headers in single submit request.
+	pub max_headers_in_single_submit: usize,
+	/// Maximal total headers size in single submit request.
+	pub max_headers_size_in_single_submit: usize,
+	/// We only may store and accept (from Ethereum node) headers that have
+	/// number >= than best_substrate_header.number - prune_depth.
+	pub prune_depth: u32,
+	/// Target transactions mode.
+	pub target_tx_mode: TargetTransactionMode,
+}
+
+/// Target transaction mode.
+#[derive(Debug, PartialEq, Clone)]
+pub enum TargetTransactionMode {
+	/// Submit new headers using signed transactions.
+	Signed,
+	/// Submit new headers using unsigned transactions.
+	Unsigned,
+	/// Submit new headers using signed transactions, but only when we
+	/// believe that sync has stalled.
+	Backup,
+}
+
+/// Headers synchronization context.
+#[derive(Debug)]
+pub struct HeadersSync<P: HeadersSyncPipeline> {
+	/// Synchronization parameters.
+	params: HeadersSyncParams,
+	/// Best header number known to source node.
+	source_best_number: Option<P::Number>,
+	/// Best header known to target node.
+	target_best_header: Option<HeaderIdOf<P>>,
+	/// Headers queue.
+	headers: QueuedHeaders<P>,
+	/// Pause headers submission.
+	pause_submit: bool,
+}
+
+impl<P: HeadersSyncPipeline> HeadersSync<P> {
+	/// Creates new headers synchronizer.
+	pub fn new(params: HeadersSyncParams) -> Self {
+		HeadersSync {
+			headers: QueuedHeaders::default(),
+			params,
+			source_best_number: None,
+			target_best_header: None,
+			pause_submit: false,
+		}
+	}
+
+	/// Return best header number known to source node.
+	pub fn source_best_number(&self) -> Option<P::Number> {
+		self.source_best_number
+	}
+
+	/// Best header known to target node.
+	pub fn target_best_header(&self) -> Option<HeaderIdOf<P>> {
+		self.target_best_header
+	}
+
+	/// Returns true if we have synced almost all known headers.
+	pub fn is_almost_synced(&self) -> bool {
+		match self.source_best_number {
+			Some(source_best_number) => self
+				.target_best_header
+				.map(|best| source_best_number.saturating_sub(best.0) < 4.into())
+				.unwrap_or(false),
+			None => true,
+		}
+	}
+
+	/// Returns synchronization status.
+	pub fn status(&self) -> (&Option<HeaderIdOf<P>>, &Option<P::Number>) {
+		(&self.target_best_header, &self.source_best_number)
+	}
+
+	/// Returns reference to the headers queue.
+	pub fn headers(&self) -> &QueuedHeaders<P> {
+		&self.headers
+	}
+
+	/// Returns mutable reference to the headers queue.
+	pub fn headers_mut(&mut self) -> &mut QueuedHeaders<P> {
+		&mut self.headers
+	}
+
+	/// Select header that needs to be downloaded from the source node.
+	pub fn select_new_header_to_download(&self) -> Option<P::Number> {
+		// if we haven't received best header from source node yet, there's nothing we can download
+		let source_best_number = self.source_best_number?;
+
+		// if we haven't received known best header from target node yet, there's nothing we can download
+		let target_best_header = self.target_best_header.as_ref()?;
+
+		// if there's too many headers in the queue, stop downloading
+		let in_memory_headers = self.headers.total_headers();
+		if in_memory_headers >= self.params.max_future_headers_to_download {
+			return None;
+		}
+
+		// if queue is empty and best header on target is > than best header on source,
+		// then we shoud reorg
+		let best_queued_number = self.headers.best_queued_number();
+		if best_queued_number.is_zero() && source_best_number < target_best_header.0 {
+			return Some(source_best_number);
+		}
+
+		// we assume that there were no reorgs if we have already downloaded best header
+		let best_downloaded_number = std::cmp::max(
+			std::cmp::max(best_queued_number, self.headers.best_synced_number()),
+			target_best_header.0,
+		);
+		if best_downloaded_number >= source_best_number {
+			return None;
+		}
+
+		// download new header
+		Some(best_downloaded_number + One::one())
+	}
+
+	/// Selech orphan header to downoload.
+	pub fn select_orphan_header_to_download(&self) -> Option<&QueuedHeader<P>> {
+		let orphan_header = self.headers.header(HeaderStatus::Orphan)?;
+
+		// we consider header orphan until we'll find it ancestor that is known to the target node
+		// => we may get orphan header while we ask target node whether it knows its parent
+		// => let's avoid fetching duplicate headers
+		let parent_id = orphan_header.parent_id();
+		if self.headers.status(&parent_id) != HeaderStatus::Unknown {
+			return None;
+		}
+
+		Some(orphan_header)
+	}
+
+	/// Select headers that need to be submitted to the target node.
+	pub fn select_headers_to_submit(&self, stalled: bool) -> Option<Vec<&QueuedHeader<P>>> {
+		// maybe we have paused new headers submit?
+		if self.pause_submit {
+			return None;
+		}
+
+		// if we operate in backup mode, we only submit headers when sync has stalled
+		if self.params.target_tx_mode == TargetTransactionMode::Backup && !stalled {
+			return None;
+		}
+
+		let headers_in_submit_status = self.headers.headers_in_status(HeaderStatus::Submitted);
+		let headers_to_submit_count = self
+			.params
+			.max_headers_in_submitted_status
+			.checked_sub(headers_in_submit_status)?;
+
+		let mut total_size = 0;
+		let mut total_headers = 0;
+		self.headers.headers(HeaderStatus::Ready, |header| {
+			if total_headers == headers_to_submit_count {
+				return false;
+			}
+			if total_headers == self.params.max_headers_in_single_submit {
+				return false;
+			}
+
+			let encoded_size = P::estimate_size(header);
+			if total_headers != 0 && total_size + encoded_size > self.params.max_headers_size_in_single_submit {
+				return false;
+			}
+
+			total_size += encoded_size;
+			total_headers += 1;
+
+			true
+		})
+	}
+
+	/// Receive new target header number from the source node.
+	pub fn source_best_header_number_response(&mut self, best_header_number: P::Number) {
+		log::debug!(
+			target: "bridge",
+			"Received best header number from {} node: {}",
+			P::SOURCE_NAME,
+			best_header_number,
+		);
+		self.source_best_number = Some(best_header_number);
+	}
+
+	/// Receive new best header from the target node.
+	/// Returns true if it is different from the previous block known to us.
+	pub fn target_best_header_response(&mut self, best_header: HeaderIdOf<P>) -> bool {
+		log::debug!(
+			target: "bridge",
+			"Received best known header from {}: {:?}",
+			P::TARGET_NAME,
+			best_header,
+		);
+
+		// early return if it is still the same
+		if self.target_best_header == Some(best_header) {
+			return false;
+		}
+
+		// remember that this header is now known to the Substrate runtime
+		self.headers.target_best_header_response(&best_header);
+
+		// prune ancient headers
+		self.headers
+			.prune(best_header.0.saturating_sub(self.params.prune_depth.into()));
+
+		// finally remember the best header itself
+		self.target_best_header = Some(best_header);
+
+		// we are ready to submit headers again
+		if self.pause_submit {
+			log::debug!(
+				target: "bridge",
+				"Ready to submit {} headers to {} node again!",
+				P::SOURCE_NAME,
+				P::TARGET_NAME,
+			);
+
+			self.pause_submit = false;
+		}
+
+		true
+	}
+
+	/// Pause headers submit until best header will be updated on target node.
+	pub fn pause_submit(&mut self) {
+		log::debug!(
+			target: "bridge",
+			"Stopping submitting {} headers to {} node. Waiting for {} submitted headers to be accepted",
+			P::SOURCE_NAME,
+			P::TARGET_NAME,
+			self.headers.headers_in_status(HeaderStatus::Submitted),
+		);
+
+		self.pause_submit = true;
+	}
+
+	/// Restart synchronization.
+	pub fn restart(&mut self) {
+		self.source_best_number = None;
+		self.target_best_header = None;
+		self.headers.clear();
+		self.pause_submit = false;
+	}
+}
+
+#[cfg(test)]
+pub mod tests {
+	use super::*;
+	use crate::headers::tests::{header, id};
+	use crate::sync_loop_tests::{TestHash, TestHeadersSyncPipeline, TestNumber};
+	use crate::sync_types::HeaderStatus;
+	use relay_utils::HeaderId;
+
+	fn side_hash(number: TestNumber) -> TestHash {
+		1000 + number
+	}
+
+	pub fn default_sync_params() -> HeadersSyncParams {
+		HeadersSyncParams {
+			max_future_headers_to_download: 128,
+			max_headers_in_submitted_status: 128,
+			max_headers_in_single_submit: 32,
+			max_headers_size_in_single_submit: 131_072,
+			prune_depth: 4096,
+			target_tx_mode: TargetTransactionMode::Signed,
+		}
+	}
+
+	#[test]
+	fn select_new_header_to_download_works() {
+		let mut eth_sync = HeadersSync::<TestHeadersSyncPipeline>::new(default_sync_params());
+
+		// both best && target headers are unknown
+		assert_eq!(eth_sync.select_new_header_to_download(), None);
+
+		// best header is known, target header is unknown
+		eth_sync.target_best_header = Some(HeaderId(0, Default::default()));
+		assert_eq!(eth_sync.select_new_header_to_download(), None);
+
+		// target header is known, best header is unknown
+		eth_sync.target_best_header = None;
+		eth_sync.source_best_number = Some(100);
+		assert_eq!(eth_sync.select_new_header_to_download(), None);
+
+		// when our best block has the same number as the target
+		eth_sync.target_best_header = Some(HeaderId(100, Default::default()));
+		assert_eq!(eth_sync.select_new_header_to_download(), None);
+
+		// when we actually need a new header
+		eth_sync.source_best_number = Some(101);
+		assert_eq!(eth_sync.select_new_header_to_download(), Some(101));
+
+		// when we have to reorganize to longer fork
+		eth_sync.source_best_number = Some(100);
+		eth_sync.target_best_header = Some(HeaderId(200, Default::default()));
+		assert_eq!(eth_sync.select_new_header_to_download(), Some(100));
+
+		// when there are too many headers scheduled for submitting
+		for i in 1..1000 {
+			eth_sync.headers.header_response(header(i).header().clone());
+		}
+		assert_eq!(eth_sync.select_new_header_to_download(), None);
+	}
+
+	#[test]
+	fn select_new_header_to_download_works_with_empty_queue() {
+		let mut eth_sync = HeadersSync::<TestHeadersSyncPipeline>::new(default_sync_params());
+		eth_sync.source_best_header_number_response(100);
+
+		// when queue is not empty => everything goes as usually
+		eth_sync.target_best_header_response(header(10).id());
+		eth_sync.headers_mut().header_response(header(11).header().clone());
+		eth_sync.headers_mut().maybe_extra_response(&header(11).id(), false);
+		assert_eq!(eth_sync.select_new_header_to_download(), Some(12));
+
+		// but then queue is drained
+		eth_sync.headers_mut().target_best_header_response(&header(11).id());
+
+		// even though it's empty, we know that header#11 is synced
+		assert_eq!(eth_sync.headers().best_queued_number(), 0);
+		assert_eq!(eth_sync.headers().best_synced_number(), 11);
+		assert_eq!(eth_sync.select_new_header_to_download(), Some(12));
+	}
+
+	#[test]
+	fn sync_without_reorgs_works() {
+		let mut eth_sync = HeadersSync::new(default_sync_params());
+		eth_sync.params.max_headers_in_submitted_status = 1;
+
+		// ethereum reports best header #102
+		eth_sync.source_best_header_number_response(102);
+
+		// substrate reports that it is at block #100
+		eth_sync.target_best_header_response(id(100));
+
+		// block #101 is downloaded first
+		assert_eq!(eth_sync.select_new_header_to_download(), Some(101));
+		eth_sync.headers.header_response(header(101).header().clone());
+
+		// now header #101 is ready to be submitted
+		assert_eq!(eth_sync.headers.header(HeaderStatus::MaybeExtra), Some(&header(101)));
+		eth_sync.headers.maybe_extra_response(&id(101), false);
+		assert_eq!(eth_sync.headers.header(HeaderStatus::Ready), Some(&header(101)));
+		assert_eq!(eth_sync.select_headers_to_submit(false), Some(vec![&header(101)]));
+
+		// and header #102 is ready to be downloaded
+		assert_eq!(eth_sync.select_new_header_to_download(), Some(102));
+		eth_sync.headers.header_response(header(102).header().clone());
+
+		// receive submission confirmation
+		eth_sync.headers.headers_submitted(vec![id(101)]);
+
+		// we have nothing to submit because previous header hasn't been confirmed yet
+		// (and we allow max 1 submit transaction in the wild)
+		assert_eq!(eth_sync.headers.header(HeaderStatus::MaybeExtra), Some(&header(102)));
+		eth_sync.headers.maybe_extra_response(&id(102), false);
+		assert_eq!(eth_sync.headers.header(HeaderStatus::Ready), Some(&header(102)));
+		assert_eq!(eth_sync.select_headers_to_submit(false), None);
+
+		// substrate reports that it has imported block #101
+		eth_sync.target_best_header_response(id(101));
+
+		// and we are ready to submit #102
+		assert_eq!(eth_sync.select_headers_to_submit(false), Some(vec![&header(102)]));
+		eth_sync.headers.headers_submitted(vec![id(102)]);
+
+		// substrate reports that it has imported block #102
+		eth_sync.target_best_header_response(id(102));
+
+		// and we have nothing to download
+		assert_eq!(eth_sync.select_new_header_to_download(), None);
+	}
+
+	#[test]
+	fn sync_with_orphan_headers_work() {
+		let mut eth_sync = HeadersSync::new(default_sync_params());
+
+		// ethereum reports best header #102
+		eth_sync.source_best_header_number_response(102);
+
+		// substrate reports that it is at block #100, but it isn't part of best chain
+		eth_sync.target_best_header_response(HeaderId(100, side_hash(100)));
+
+		// block #101 is downloaded first
+		assert_eq!(eth_sync.select_new_header_to_download(), Some(101));
+		eth_sync.headers.header_response(header(101).header().clone());
+
+		// we can't submit header #101, because its parent status is unknown
+		assert_eq!(eth_sync.select_headers_to_submit(false), None);
+
+		// instead we are trying to determine status of its parent (#100)
+		assert_eq!(eth_sync.headers.header(HeaderStatus::MaybeOrphan), Some(&header(101)));
+
+		// and the status is still unknown
+		eth_sync.headers.maybe_orphan_response(&id(100), false);
+
+		// so we consider #101 orphaned now && will download its parent - #100
+		assert_eq!(eth_sync.headers.header(HeaderStatus::Orphan), Some(&header(101)));
+		eth_sync.headers.header_response(header(100).header().clone());
+
+		// #101 is now Orphan and #100 is MaybeOrphan => we do not want to retrieve
+		// header #100 again
+		assert_eq!(eth_sync.headers.header(HeaderStatus::Orphan), Some(&header(101)));
+		assert_eq!(eth_sync.select_orphan_header_to_download(), None);
+
+		// we can't submit header #100, because its parent status is unknown
+		assert_eq!(eth_sync.select_headers_to_submit(false), None);
+
+		// instead we are trying to determine status of its parent (#99)
+		assert_eq!(eth_sync.headers.header(HeaderStatus::MaybeOrphan), Some(&header(100)));
+
+		// and the status is known, so we move previously orphaned #100 and #101 to ready queue
+		eth_sync.headers.maybe_orphan_response(&id(99), true);
+
+		// and we are ready to submit #100
+		assert_eq!(eth_sync.headers.header(HeaderStatus::MaybeExtra), Some(&header(100)));
+		eth_sync.headers.maybe_extra_response(&id(100), false);
+		assert_eq!(eth_sync.select_headers_to_submit(false), Some(vec![&header(100)]));
+		eth_sync.headers.headers_submitted(vec![id(100)]);
+
+		// and we are ready to submit #101
+		assert_eq!(eth_sync.headers.header(HeaderStatus::MaybeExtra), Some(&header(101)));
+		eth_sync.headers.maybe_extra_response(&id(101), false);
+		assert_eq!(eth_sync.select_headers_to_submit(false), Some(vec![&header(101)]));
+		eth_sync.headers.headers_submitted(vec![id(101)]);
+	}
+
+	#[test]
+	fn pruning_happens_on_target_best_header_response() {
+		let mut eth_sync = HeadersSync::<TestHeadersSyncPipeline>::new(default_sync_params());
+		eth_sync.params.prune_depth = 50;
+		eth_sync.target_best_header_response(id(100));
+		assert_eq!(eth_sync.headers.prune_border(), 50);
+	}
+
+	#[test]
+	fn only_submitting_headers_in_backup_mode_when_stalled() {
+		let mut eth_sync = HeadersSync::new(default_sync_params());
+		eth_sync.params.target_tx_mode = TargetTransactionMode::Backup;
+
+		// ethereum reports best header #102
+		eth_sync.source_best_header_number_response(102);
+
+		// substrate reports that it is at block #100
+		eth_sync.target_best_header_response(id(100));
+
+		// block #101 is downloaded first
+		eth_sync.headers.header_response(header(101).header().clone());
+		eth_sync.headers.maybe_extra_response(&id(101), false);
+
+		// ensure that headers are not submitted when sync is not stalled
+		assert_eq!(eth_sync.select_headers_to_submit(false), None);
+
+		// ensure that headers are not submitted when sync is stalled
+		assert_eq!(eth_sync.select_headers_to_submit(true), Some(vec![&header(101)]));
+	}
+
+	#[test]
+	fn does_not_select_new_headers_to_submit_when_submit_is_paused() {
+		let mut eth_sync = HeadersSync::new(default_sync_params());
+		eth_sync.params.max_headers_in_submitted_status = 1;
+
+		// ethereum reports best header #102 and substrate is at #100
+		eth_sync.source_best_header_number_response(102);
+		eth_sync.target_best_header_response(id(100));
+
+		// let's prepare #101 and #102 for submitting
+		eth_sync.headers.header_response(header(101).header().clone());
+		eth_sync.headers.maybe_extra_response(&id(101), false);
+		eth_sync.headers.header_response(header(102).header().clone());
+		eth_sync.headers.maybe_extra_response(&id(102), false);
+
+		// when submit is not paused, we're ready to submit #101
+		assert_eq!(eth_sync.select_headers_to_submit(false), Some(vec![&header(101)]));
+
+		// when submit is paused, we're not ready to submit anything
+		eth_sync.pause_submit();
+		assert_eq!(eth_sync.select_headers_to_submit(false), None);
+
+		// if best header on substrate node isn't updated, we still not submitting anything
+		eth_sync.target_best_header_response(id(100));
+		assert_eq!(eth_sync.select_headers_to_submit(false), None);
+
+		// but after it is actually updated, we are ready to submit
+		eth_sync.target_best_header_response(id(101));
+		assert_eq!(eth_sync.select_headers_to_submit(false), Some(vec![&header(102)]));
+	}
+}

bridges/relays/generic/headers/src/sync_loop.rs

@@ -0,0 +1,654 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Entrypoint for running headers synchronization loop.
+
+use crate::sync::{HeadersSync, HeadersSyncParams};
+use crate::sync_loop_metrics::SyncLoopMetrics;
+use crate::sync_types::{HeaderIdOf, HeaderStatus, HeadersSyncPipeline, QueuedHeader, SubmittedHeaders};
+
+use async_trait::async_trait;
+use futures::{future::FutureExt, stream::StreamExt};
+use num_traits::{Saturating, Zero};
+use relay_utils::{
+	format_ids, interval,
+	metrics::{start as metrics_start, GlobalMetrics, MetricsParams},
+	process_future_result,
+	relay_loop::Client as RelayClient,
+	retry_backoff, FailedClient, MaybeConnectionError, StringifiedMaybeConnectionError,
+};
+use std::{
+	collections::HashSet,
+	future::Future,
+	time::{Duration, Instant},
+};
+
+/// When we submit headers to target node, but see no updates of best
+/// source block known to target node during STALL_SYNC_TIMEOUT seconds,
+/// we consider that our headers are rejected because there has been reorg in target chain.
+/// This reorg could invalidate our knowledge about sync process (i.e. we have asked if
+/// HeaderA is known to target, but then reorg happened and the answer is different
+/// now) => we need to reset sync.
+/// The other option is to receive **EVERY** best target header and check if it is
+/// direct child of previous best header. But: (1) subscription doesn't guarantee that
+/// the subscriber will receive every best header (2) reorg won't always lead to sync
+/// stall and restart is a heavy operation (we forget all in-memory headers).
+const STALL_SYNC_TIMEOUT: Duration = Duration::from_secs(5 * 60);
+/// Delay after we have seen update of best source header at target node,
+/// for us to treat sync stalled. ONLY when relay operates in backup mode.
+const BACKUP_STALL_SYNC_TIMEOUT: Duration = Duration::from_secs(10 * 60);
+/// Interval between calling sync maintain procedure.
+const MAINTAIN_INTERVAL: Duration = Duration::from_secs(30);
+
+/// Source client trait.
+#[async_trait]
+pub trait SourceClient<P: HeadersSyncPipeline>: RelayClient {
+	/// Get best block number.
+	async fn best_block_number(&self) -> Result<P::Number, Self::Error>;
+
+	/// Get header by hash.
+	async fn header_by_hash(&self, hash: P::Hash) -> Result<P::Header, Self::Error>;
+
+	/// Get canonical header by number.
+	async fn header_by_number(&self, number: P::Number) -> Result<P::Header, Self::Error>;
+
+	/// Get completion data by header hash.
+	async fn header_completion(&self, id: HeaderIdOf<P>)
+		-> Result<(HeaderIdOf<P>, Option<P::Completion>), Self::Error>;
+
+	/// Get extra data by header hash.
+	async fn header_extra(
+		&self,
+		id: HeaderIdOf<P>,
+		header: QueuedHeader<P>,
+	) -> Result<(HeaderIdOf<P>, P::Extra), Self::Error>;
+}
+
+/// Target client trait.
+#[async_trait]
+pub trait TargetClient<P: HeadersSyncPipeline>: RelayClient {
+	/// Returns ID of best header known to the target node.
+	async fn best_header_id(&self) -> Result<HeaderIdOf<P>, Self::Error>;
+
+	/// Returns true if header is known to the target node.
+	async fn is_known_header(&self, id: HeaderIdOf<P>) -> Result<(HeaderIdOf<P>, bool), Self::Error>;
+
+	/// Submit headers.
+	async fn submit_headers(&self, headers: Vec<QueuedHeader<P>>) -> SubmittedHeaders<HeaderIdOf<P>, Self::Error>;
+
+	/// Returns ID of headers that require to be 'completed' before children can be submitted.
+	async fn incomplete_headers_ids(&self) -> Result<HashSet<HeaderIdOf<P>>, Self::Error>;
+
+	/// Submit completion data for header.
+	async fn complete_header(&self, id: HeaderIdOf<P>, completion: P::Completion)
+		-> Result<HeaderIdOf<P>, Self::Error>;
+
+	/// Returns true if header requires extra data to be submitted.
+	async fn requires_extra(&self, header: QueuedHeader<P>) -> Result<(HeaderIdOf<P>, bool), Self::Error>;
+}
+
+/// Synchronization maintain procedure.
+#[async_trait]
+pub trait SyncMaintain<P: HeadersSyncPipeline>: Clone + Send + Sync {
+	/// Run custom maintain procedures. This is guaranteed to be called when both source and target
+	/// clients are unoccupied.
+	async fn maintain(&self, _sync: &mut HeadersSync<P>) {}
+}
+
+impl<P: HeadersSyncPipeline> SyncMaintain<P> for () {}
+
+/// Run headers synchronization.
+#[allow(clippy::too_many_arguments)]
+pub fn run<P: HeadersSyncPipeline, TC: TargetClient<P>>(
+	source_client: impl SourceClient<P>,
+	source_tick: Duration,
+	target_client: TC,
+	target_tick: Duration,
+	sync_maintain: impl SyncMaintain<P>,
+	sync_params: HeadersSyncParams,
+	metrics_params: Option<MetricsParams>,
+	exit_signal: impl Future<Output = ()>,
+) {
+	let exit_signal = exit_signal.shared();
+
+	let metrics_global = GlobalMetrics::default();
+	let metrics_sync = SyncLoopMetrics::default();
+	let metrics_enabled = metrics_params.is_some();
+	metrics_start(
+		format!("{}_to_{}_Sync", P::SOURCE_NAME, P::TARGET_NAME),
+		metrics_params,
+		&metrics_global,
+		&metrics_sync,
+	);
+
+	relay_utils::relay_loop::run(
+		relay_utils::relay_loop::RECONNECT_DELAY,
+		source_client,
+		target_client,
+		|source_client, target_client| {
+			run_until_connection_lost(
+				source_client,
+				source_tick,
+				target_client,
+				target_tick,
+				sync_maintain.clone(),
+				sync_params.clone(),
+				if metrics_enabled {
+					Some(metrics_global.clone())
+				} else {
+					None
+				},
+				if metrics_enabled {
+					Some(metrics_sync.clone())
+				} else {
+					None
+				},
+				exit_signal.clone(),
+			)
+		},
+	);
+}
+
+/// Run headers synchronization.
+#[allow(clippy::too_many_arguments)]
+async fn run_until_connection_lost<P: HeadersSyncPipeline, TC: TargetClient<P>>(
+	source_client: impl SourceClient<P>,
+	source_tick: Duration,
+	target_client: TC,
+	target_tick: Duration,
+	sync_maintain: impl SyncMaintain<P>,
+	sync_params: HeadersSyncParams,
+	metrics_global: Option<GlobalMetrics>,
+	metrics_sync: Option<SyncLoopMetrics>,
+	exit_signal: impl Future<Output = ()>,
+) -> Result<(), FailedClient> {
+	let mut progress_context = (Instant::now(), None, None);
+
+	let mut sync = HeadersSync::<P>::new(sync_params);
+	let mut stall_countdown = None;
+	let mut last_update_time = Instant::now();
+
+	let mut source_retry_backoff = retry_backoff();
+	let mut source_client_is_online = false;
+	let mut source_best_block_number_required = false;
+	let source_best_block_number_future = source_client.best_block_number().fuse();
+	let source_new_header_future = futures::future::Fuse::terminated();
+	let source_orphan_header_future = futures::future::Fuse::terminated();
+	let source_extra_future = futures::future::Fuse::terminated();
+	let source_completion_future = futures::future::Fuse::terminated();
+	let source_go_offline_future = futures::future::Fuse::terminated();
+	let source_tick_stream = interval(source_tick).fuse();
+
+	let mut target_retry_backoff = retry_backoff();
+	let mut target_client_is_online = false;
+	let mut target_best_block_required = false;
+	let mut target_incomplete_headers_required = true;
+	let target_best_block_future = target_client.best_header_id().fuse();
+	let target_incomplete_headers_future = futures::future::Fuse::terminated();
+	let target_extra_check_future = futures::future::Fuse::terminated();
+	let target_existence_status_future = futures::future::Fuse::terminated();
+	let target_submit_header_future = futures::future::Fuse::terminated();
+	let target_complete_header_future = futures::future::Fuse::terminated();
+	let target_go_offline_future = futures::future::Fuse::terminated();
+	let target_tick_stream = interval(target_tick).fuse();
+
+	let mut maintain_required = false;
+	let maintain_stream = interval(MAINTAIN_INTERVAL).fuse();
+
+	let exit_signal = exit_signal.fuse();
+
+	futures::pin_mut!(
+		source_best_block_number_future,
+		source_new_header_future,
+		source_orphan_header_future,
+		source_extra_future,
+		source_completion_future,
+		source_go_offline_future,
+		source_tick_stream,
+		target_best_block_future,
+		target_incomplete_headers_future,
+		target_extra_check_future,
+		target_existence_status_future,
+		target_submit_header_future,
+		target_complete_header_future,
+		target_go_offline_future,
+		target_tick_stream,
+		maintain_stream,
+		exit_signal
+	);
+
+	loop {
+		futures::select! {
+			source_best_block_number = source_best_block_number_future => {
+				source_best_block_number_required = false;
+
+				source_client_is_online = process_future_result(
+					source_best_block_number,
+					&mut source_retry_backoff,
+					|source_best_block_number| sync.source_best_header_number_response(source_best_block_number),
+					&mut source_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving best header number from {}", P::SOURCE_NAME),
+				).fail_if_connection_error(FailedClient::Source)?;
+			},
+			source_new_header = source_new_header_future => {
+				source_client_is_online = process_future_result(
+					source_new_header,
+					&mut source_retry_backoff,
+					|source_new_header| sync.headers_mut().header_response(source_new_header),
+					&mut source_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving header from {} node", P::SOURCE_NAME),
+				).fail_if_connection_error(FailedClient::Source)?;
+			},
+			source_orphan_header = source_orphan_header_future => {
+				source_client_is_online = process_future_result(
+					source_orphan_header,
+					&mut source_retry_backoff,
+					|source_orphan_header| sync.headers_mut().header_response(source_orphan_header),
+					&mut source_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving orphan header from {} node", P::SOURCE_NAME),
+				).fail_if_connection_error(FailedClient::Source)?;
+			},
+			source_extra = source_extra_future => {
+				source_client_is_online = process_future_result(
+					source_extra,
+					&mut source_retry_backoff,
+					|(header, extra)| sync.headers_mut().extra_response(&header, extra),
+					&mut source_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving extra data from {} node", P::SOURCE_NAME),
+				).fail_if_connection_error(FailedClient::Source)?;
+			},
+			source_completion = source_completion_future => {
+				source_client_is_online = process_future_result(
+					source_completion,
+					&mut source_retry_backoff,
+					|(header, completion)| sync.headers_mut().completion_response(&header, completion),
+					&mut source_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving completion data from {} node", P::SOURCE_NAME),
+				).fail_if_connection_error(FailedClient::Source)?;
+			},
+			_ = source_go_offline_future => {
+				source_client_is_online = true;
+			},
+			_ = source_tick_stream.next() => {
+				if sync.is_almost_synced() {
+					source_best_block_number_required = true;
+				}
+			},
+			target_best_block = target_best_block_future => {
+				target_best_block_required = false;
+
+				target_client_is_online = process_future_result(
+					target_best_block,
+					&mut target_retry_backoff,
+					|target_best_block| {
+						let head_updated = sync.target_best_header_response(target_best_block);
+						if head_updated {
+							last_update_time = Instant::now();
+						}
+						match head_updated {
+							// IF head is updated AND there are still our transactions:
+							// => restart stall countdown timer
+							true if sync.headers().headers_in_status(HeaderStatus::Submitted) != 0 =>
+								stall_countdown = Some(Instant::now()),
+							// IF head is updated AND there are no our transactions:
+							// => stop stall countdown timer
+							true => stall_countdown = None,
+							// IF head is not updated AND stall countdown is not yet completed
+							// => do nothing
+							false if stall_countdown
+								.map(|stall_countdown| stall_countdown.elapsed() < STALL_SYNC_TIMEOUT)
+								.unwrap_or(true)
+								=> (),
+							// IF head is not updated AND stall countdown has completed
+							// => restart sync
+							false => {
+								log::info!(
+									target: "bridge",
+									"Sync has stalled. Restarting {} headers synchronization.",
+									P::SOURCE_NAME,
+								);
+								stall_countdown = None;
+								sync.restart();
+							},
+						}
+					},
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving best known {} header from {} node", P::SOURCE_NAME, P::TARGET_NAME),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+			incomplete_headers_ids = target_incomplete_headers_future => {
+				target_incomplete_headers_required = false;
+
+				target_client_is_online = process_future_result(
+					incomplete_headers_ids,
+					&mut target_retry_backoff,
+					|incomplete_headers_ids| sync.headers_mut().incomplete_headers_response(incomplete_headers_ids),
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving incomplete headers from {} node", P::TARGET_NAME),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+			target_existence_status = target_existence_status_future => {
+				target_client_is_online = process_future_result(
+					target_existence_status,
+					&mut target_retry_backoff,
+					|(target_header, target_existence_status)| sync
+						.headers_mut()
+						.maybe_orphan_response(&target_header, target_existence_status),
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving existence status from {} node", P::TARGET_NAME),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+			submitted_headers = target_submit_header_future => {
+				// following line helps Rust understand the type of `submitted_headers` :/
+				let submitted_headers: SubmittedHeaders<HeaderIdOf<P>, TC::Error> = submitted_headers;
+				let submitted_headers_str = format!("{}", submitted_headers);
+				let all_headers_rejected = submitted_headers.submitted.is_empty()
+					&& submitted_headers.incomplete.is_empty();
+				let has_submitted_headers = sync.headers().headers_in_status(HeaderStatus::Submitted) != 0;
+
+				let maybe_fatal_error = match submitted_headers.fatal_error {
+					Some(fatal_error) => Err(StringifiedMaybeConnectionError::new(
+						fatal_error.is_connection_error(),
+						format!("{:?}", fatal_error),
+					)),
+					None if all_headers_rejected && !has_submitted_headers =>
+						Err(StringifiedMaybeConnectionError::new(false, "All headers were rejected".into())),
+					None => Ok(()),
+				};
+
+				let no_fatal_error = maybe_fatal_error.is_ok();
+				target_client_is_online = process_future_result(
+					maybe_fatal_error,
+					&mut target_retry_backoff,
+					|_| {},
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error submitting headers to {} node", P::TARGET_NAME),
+				).fail_if_connection_error(FailedClient::Target)?;
+
+				log::debug!(target: "bridge", "Header submit result: {}", submitted_headers_str);
+
+				sync.headers_mut().headers_submitted(submitted_headers.submitted);
+				sync.headers_mut().add_incomplete_headers(false, submitted_headers.incomplete);
+
+				// when there's no fatal error, but node has rejected all our headers we may
+				// want to pause until our submitted headers will be accepted
+				if no_fatal_error && all_headers_rejected && has_submitted_headers {
+					sync.pause_submit();
+				}
+			},
+			target_complete_header_result = target_complete_header_future => {
+				target_client_is_online = process_future_result(
+					target_complete_header_result,
+					&mut target_retry_backoff,
+					|completed_header| sync.headers_mut().header_completed(&completed_header),
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error completing headers at {}", P::TARGET_NAME),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+			target_extra_check_result = target_extra_check_future => {
+				target_client_is_online = process_future_result(
+					target_extra_check_result,
+					&mut target_retry_backoff,
+					|(header, extra_check_result)| sync
+						.headers_mut()
+						.maybe_extra_response(&header, extra_check_result),
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving receipts requirement from {} node", P::TARGET_NAME),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+			_ = target_go_offline_future => {
+				target_client_is_online = true;
+			},
+			_ = target_tick_stream.next() => {
+				target_best_block_required = true;
+				target_incomplete_headers_required = true;
+			},
+
+			_ = maintain_stream.next() => {
+				maintain_required = true;
+			},
+			_ = exit_signal => {
+				return Ok(());
+			}
+		}
+
+		// update metrics
+		if let Some(ref metrics_global) = metrics_global {
+			metrics_global.update().await;
+		}
+		if let Some(ref metrics_sync) = metrics_sync {
+			metrics_sync.update(&sync);
+		}
+
+		// print progress
+		progress_context = print_sync_progress(progress_context, &sync);
+
+		// run maintain procedures
+		if maintain_required && source_client_is_online && target_client_is_online {
+			log::debug!(target: "bridge", "Maintaining headers sync loop");
+			maintain_required = false;
+			sync_maintain.maintain(&mut sync).await;
+		}
+
+		// If the target client is accepting requests we update the requests that
+		// we want it to run
+		if !maintain_required && target_client_is_online {
+			// NOTE: Is is important to reset this so that we only have one
+			// request being processed by the client at a time. This prevents
+			// race conditions like receiving two transactions with the same
+			// nonce from the client.
+			target_client_is_online = false;
+
+			// The following is how we prioritize requests:
+			//
+			// 1. Get best block
+			//     - Stops us from downloading or submitting new blocks
+			//     - Only called rarely
+			//
+			// 2. Get incomplete headers
+			//     - Stops us from submitting new blocks
+			//     - Only called rarely
+			//
+			// 3. Get complete headers
+			//     - Stops us from submitting new blocks
+			//
+			// 4. Check if we need extra data from source
+			//     - Stops us from downloading or submitting new blocks
+			//
+			// 5. Check existence of header
+			//     - Stops us from submitting new blocks
+			//
+			// 6. Submit header
+
+			if target_best_block_required {
+				log::debug!(target: "bridge", "Asking {} about best block", P::TARGET_NAME);
+				target_best_block_future.set(target_client.best_header_id().fuse());
+			} else if target_incomplete_headers_required {
+				log::debug!(target: "bridge", "Asking {} about incomplete headers", P::TARGET_NAME);
+				target_incomplete_headers_future.set(target_client.incomplete_headers_ids().fuse());
+			} else if let Some((id, completion)) = sync.headers_mut().header_to_complete() {
+				log::debug!(
+					target: "bridge",
+					"Going to complete header: {:?}",
+					id,
+				);
+
+				target_complete_header_future.set(target_client.complete_header(id, completion.clone()).fuse());
+			} else if let Some(header) = sync.headers().header(HeaderStatus::MaybeExtra) {
+				log::debug!(
+					target: "bridge",
+					"Checking if header submission requires extra: {:?}",
+					header.id(),
+				);
+
+				target_extra_check_future.set(target_client.requires_extra(header.clone()).fuse());
+			} else if let Some(header) = sync.headers().header(HeaderStatus::MaybeOrphan) {
+				// for MaybeOrphan we actually ask for parent' header existence
+				let parent_id = header.parent_id();
+
+				log::debug!(
+					target: "bridge",
+					"Asking {} node for existence of: {:?}",
+					P::TARGET_NAME,
+					parent_id,
+				);
+
+				target_existence_status_future.set(target_client.is_known_header(parent_id).fuse());
+			} else if let Some(headers) =
+				sync.select_headers_to_submit(last_update_time.elapsed() > BACKUP_STALL_SYNC_TIMEOUT)
+			{
+				log::debug!(
+					target: "bridge",
+					"Submitting {} header(s) to {} node: {:?}",
+					headers.len(),
+					P::TARGET_NAME,
+					format_ids(headers.iter().map(|header| header.id())),
+				);
+
+				let headers = headers.into_iter().cloned().collect();
+				target_submit_header_future.set(target_client.submit_headers(headers).fuse());
+
+				// remember that we have submitted some headers
+				if stall_countdown.is_none() {
+					stall_countdown = Some(Instant::now());
+				}
+			} else {
+				target_client_is_online = true;
+			}
+		}
+
+		// If the source client is accepting requests we update the requests that
+		// we want it to run
+		if !maintain_required && source_client_is_online {
+			// NOTE: Is is important to reset this so that we only have one
+			// request being processed by the client at a time. This prevents
+			// race conditions like receiving two transactions with the same
+			// nonce from the client.
+			source_client_is_online = false;
+
+			// The following is how we prioritize requests:
+			//
+			// 1. Get best block
+			//     - Stops us from downloading or submitting new blocks
+			//     - Only called rarely
+			//
+			// 2. Download completion data
+			//     - Stops us from submitting new blocks
+			//
+			// 3. Download extra data
+			//     - Stops us from submitting new blocks
+			//
+			// 4. Download missing headers
+			//     - Stops us from downloading or submitting new blocks
+			//
+			// 5. Downloading new headers
+
+			if source_best_block_number_required {
+				log::debug!(target: "bridge", "Asking {} node about best block number", P::SOURCE_NAME);
+				source_best_block_number_future.set(source_client.best_block_number().fuse());
+			} else if let Some(id) = sync.headers_mut().incomplete_header() {
+				log::debug!(
+					target: "bridge",
+					"Retrieving completion data for header: {:?}",
+					id,
+				);
+				source_completion_future.set(source_client.header_completion(id).fuse());
+			} else if let Some(header) = sync.headers().header(HeaderStatus::Extra) {
+				let id = header.id();
+				log::debug!(
+					target: "bridge",
+					"Retrieving extra data for header: {:?}",
+					id,
+				);
+				source_extra_future.set(source_client.header_extra(id, header.clone()).fuse());
+			} else if let Some(header) = sync.select_orphan_header_to_download() {
+				// for Orphan we actually ask for parent' header
+				let parent_id = header.parent_id();
+
+				// if we have end up with orphan header#0, then we are misconfigured
+				if parent_id.0.is_zero() {
+					log::error!(
+						target: "bridge",
+						"Misconfiguration. Genesis {} header is considered orphan by {} node",
+						P::SOURCE_NAME,
+						P::TARGET_NAME,
+					);
+					return Ok(());
+				}
+
+				log::debug!(
+					target: "bridge",
+					"Going to download orphan header from {} node: {:?}",
+					P::SOURCE_NAME,
+					parent_id,
+				);
+
+				source_orphan_header_future.set(source_client.header_by_hash(parent_id.1).fuse());
+			} else if let Some(id) = sync.select_new_header_to_download() {
+				log::debug!(
+					target: "bridge",
+					"Going to download new header from {} node: {:?}",
+					P::SOURCE_NAME,
+					id,
+				);
+
+				source_new_header_future.set(source_client.header_by_number(id).fuse());
+			} else {
+				source_client_is_online = true;
+			}
+		}
+	}
+}
+
+/// Print synchronization progress.
+fn print_sync_progress<P: HeadersSyncPipeline>(
+	progress_context: (Instant, Option<P::Number>, Option<P::Number>),
+	eth_sync: &HeadersSync<P>,
+) -> (Instant, Option<P::Number>, Option<P::Number>) {
+	let (prev_time, prev_best_header, prev_target_header) = progress_context;
+	let now_time = Instant::now();
+	let (now_best_header, now_target_header) = eth_sync.status();
+
+	let need_update = now_time - prev_time > Duration::from_secs(10)
+		|| match (prev_best_header, now_best_header) {
+			(Some(prev_best_header), Some(now_best_header)) => {
+				now_best_header.0.saturating_sub(prev_best_header) > 10.into()
+			}
+			_ => false,
+		};
+	if !need_update {
+		return (prev_time, prev_best_header, prev_target_header);
+	}
+
+	log::info!(
+		target: "bridge",
+		"Synced {:?} of {:?} headers",
+		now_best_header.map(|id| id.0),
+		now_target_header,
+	);
+	(now_time, now_best_header.clone().map(|id| id.0), *now_target_header)
+}

bridges/relays/generic/headers/src/sync_loop_metrics.rs

@@ -0,0 +1,105 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Metrics for headers synchronization relay loop.
+
+use crate::sync::HeadersSync;
+use crate::sync_types::{HeaderStatus, HeadersSyncPipeline};
+
+use num_traits::Zero;
+use relay_utils::metrics::{register, GaugeVec, Metrics, Opts, Registry, U64};
+
+/// Headers sync metrics.
+#[derive(Clone)]
+pub struct SyncLoopMetrics {
+	/// Best syncing headers at "source" and "target" nodes.
+	best_block_numbers: GaugeVec<U64>,
+	/// Number of headers in given states (see `HeaderStatus`).
+	blocks_in_state: GaugeVec<U64>,
+}
+
+impl Metrics for SyncLoopMetrics {
+	fn register(&self, registry: &Registry) -> Result<(), String> {
+		register(self.best_block_numbers.clone(), registry).map_err(|e| e.to_string())?;
+		register(self.blocks_in_state.clone(), registry).map_err(|e| e.to_string())?;
+		Ok(())
+	}
+}
+
+impl Default for SyncLoopMetrics {
+	fn default() -> Self {
+		SyncLoopMetrics {
+			best_block_numbers: GaugeVec::new(
+				Opts::new("best_block_numbers", "Best block numbers on source and target nodes"),
+				&["node"],
+			)
+			.expect("metric is static and thus valid; qed"),
+			blocks_in_state: GaugeVec::new(
+				Opts::new("blocks_in_state", "Number of blocks in given state"),
+				&["state"],
+			)
+			.expect("metric is static and thus valid; qed"),
+		}
+	}
+}
+
+impl SyncLoopMetrics {
+	/// Update best block number at source.
+	pub fn update_best_block_at_source<Number: Into<u64>>(&self, source_best_number: Number) {
+		self.best_block_numbers
+			.with_label_values(&["source"])
+			.set(source_best_number.into());
+	}
+
+	/// Update best block number at target.
+	pub fn update_best_block_at_target<Number: Into<u64>>(&self, target_best_number: Number) {
+		self.best_block_numbers
+			.with_label_values(&["target"])
+			.set(target_best_number.into());
+	}
+
+	/// Update metrics.
+	pub fn update<P: HeadersSyncPipeline>(&self, sync: &HeadersSync<P>) {
+		let headers = sync.headers();
+		let source_best_number = sync.source_best_number().unwrap_or_else(Zero::zero);
+		let target_best_number = sync.target_best_header().map(|id| id.0).unwrap_or_else(Zero::zero);
+
+		self.update_best_block_at_source(source_best_number);
+		self.update_best_block_at_target(target_best_number);
+
+		self.blocks_in_state
+			.with_label_values(&["maybe_orphan"])
+			.set(headers.headers_in_status(HeaderStatus::MaybeOrphan) as _);
+		self.blocks_in_state
+			.with_label_values(&["orphan"])
+			.set(headers.headers_in_status(HeaderStatus::Orphan) as _);
+		self.blocks_in_state
+			.with_label_values(&["maybe_extra"])
+			.set(headers.headers_in_status(HeaderStatus::MaybeExtra) as _);
+		self.blocks_in_state
+			.with_label_values(&["extra"])
+			.set(headers.headers_in_status(HeaderStatus::Extra) as _);
+		self.blocks_in_state
+			.with_label_values(&["ready"])
+			.set(headers.headers_in_status(HeaderStatus::Ready) as _);
+		self.blocks_in_state
+			.with_label_values(&["incomplete"])
+			.set(headers.headers_in_status(HeaderStatus::Incomplete) as _);
+		self.blocks_in_state
+			.with_label_values(&["submitted"])
+			.set(headers.headers_in_status(HeaderStatus::Submitted) as _);
+	}
+}

bridges/relays/generic/headers/src/sync_loop_tests.rs

@@ -0,0 +1,593 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+#![cfg(test)]
+
+use crate::sync_loop::{run, SourceClient, TargetClient};
+use crate::sync_types::{HeadersSyncPipeline, QueuedHeader, SourceHeader, SubmittedHeaders};
+
+use async_trait::async_trait;
+use backoff::backoff::Backoff;
+use futures::{future::FutureExt, stream::StreamExt};
+use parking_lot::Mutex;
+use relay_utils::{
+	process_future_result, relay_loop::Client as RelayClient, retry_backoff, HeaderId, MaybeConnectionError,
+};
+use std::{
+	collections::{HashMap, HashSet},
+	sync::Arc,
+	time::Duration,
+};
+
+pub type TestNumber = u64;
+pub type TestHash = u64;
+pub type TestHeaderId = HeaderId<TestHash, TestNumber>;
+pub type TestExtra = u64;
+pub type TestCompletion = u64;
+pub type TestQueuedHeader = QueuedHeader<TestHeadersSyncPipeline>;
+
+#[derive(Default, Debug, Clone, PartialEq)]
+pub struct TestHeader {
+	pub hash: TestHash,
+	pub number: TestNumber,
+	pub parent_hash: TestHash,
+}
+
+impl SourceHeader<TestHash, TestNumber> for TestHeader {
+	fn id(&self) -> TestHeaderId {
+		HeaderId(self.number, self.hash)
+	}
+
+	fn parent_id(&self) -> TestHeaderId {
+		HeaderId(self.number - 1, self.parent_hash)
+	}
+}
+
+#[derive(Debug, Clone)]
+struct TestError(bool);
+
+impl MaybeConnectionError for TestError {
+	fn is_connection_error(&self) -> bool {
+		self.0
+	}
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct TestHeadersSyncPipeline;
+
+impl HeadersSyncPipeline for TestHeadersSyncPipeline {
+	const SOURCE_NAME: &'static str = "Source";
+	const TARGET_NAME: &'static str = "Target";
+
+	type Hash = TestHash;
+	type Number = TestNumber;
+	type Header = TestHeader;
+	type Extra = TestExtra;
+	type Completion = TestCompletion;
+
+	fn estimate_size(_: &TestQueuedHeader) -> usize {
+		0
+	}
+}
+
+enum SourceMethod {
+	BestBlockNumber,
+	HeaderByHash(TestHash),
+	HeaderByNumber(TestNumber),
+	HeaderCompletion(TestHeaderId),
+	HeaderExtra(TestHeaderId, TestQueuedHeader),
+}
+
+#[derive(Clone)]
+struct Source {
+	data: Arc<Mutex<SourceData>>,
+	on_method_call: Arc<dyn Fn(SourceMethod, &mut SourceData) + Send + Sync>,
+}
+
+struct SourceData {
+	best_block_number: Result<TestNumber, TestError>,
+	header_by_hash: HashMap<TestHash, TestHeader>,
+	header_by_number: HashMap<TestNumber, TestHeader>,
+	provides_completion: bool,
+	provides_extra: bool,
+}
+
+impl Source {
+	pub fn new(
+		best_block_id: TestHeaderId,
+		headers: Vec<(bool, TestHeader)>,
+		on_method_call: impl Fn(SourceMethod, &mut SourceData) + Send + Sync + 'static,
+	) -> Self {
+		Source {
+			data: Arc::new(Mutex::new(SourceData {
+				best_block_number: Ok(best_block_id.0),
+				header_by_hash: headers
+					.iter()
+					.map(|(_, header)| (header.hash, header.clone()))
+					.collect(),
+				header_by_number: headers
+					.iter()
+					.filter_map(|(is_canonical, header)| {
+						if *is_canonical {
+							Some((header.hash, header.clone()))
+						} else {
+							None
+						}
+					})
+					.collect(),
+				provides_completion: true,
+				provides_extra: true,
+			})),
+			on_method_call: Arc::new(on_method_call),
+		}
+	}
+}
+
+#[async_trait]
+impl RelayClient for Source {
+	type Error = TestError;
+
+	async fn reconnect(&mut self) -> Result<(), TestError> {
+		unimplemented!()
+	}
+}
+
+#[async_trait]
+impl SourceClient<TestHeadersSyncPipeline> for Source {
+	async fn best_block_number(&self) -> Result<TestNumber, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(SourceMethod::BestBlockNumber, &mut *data);
+		data.best_block_number.clone()
+	}
+
+	async fn header_by_hash(&self, hash: TestHash) -> Result<TestHeader, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(SourceMethod::HeaderByHash(hash), &mut *data);
+		data.header_by_hash.get(&hash).cloned().ok_or(TestError(false))
+	}
+
+	async fn header_by_number(&self, number: TestNumber) -> Result<TestHeader, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(SourceMethod::HeaderByNumber(number), &mut *data);
+		data.header_by_number.get(&number).cloned().ok_or(TestError(false))
+	}
+
+	async fn header_completion(&self, id: TestHeaderId) -> Result<(TestHeaderId, Option<TestCompletion>), TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(SourceMethod::HeaderCompletion(id), &mut *data);
+		if data.provides_completion {
+			Ok((id, Some(test_completion(id))))
+		} else {
+			Ok((id, None))
+		}
+	}
+
+	async fn header_extra(
+		&self,
+		id: TestHeaderId,
+		header: TestQueuedHeader,
+	) -> Result<(TestHeaderId, TestExtra), TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(SourceMethod::HeaderExtra(id, header), &mut *data);
+		if data.provides_extra {
+			Ok((id, test_extra(id)))
+		} else {
+			Err(TestError(false))
+		}
+	}
+}
+
+enum TargetMethod {
+	BestHeaderId,
+	IsKnownHeader(TestHeaderId),
+	SubmitHeaders(Vec<TestQueuedHeader>),
+	IncompleteHeadersIds,
+	CompleteHeader(TestHeaderId, TestCompletion),
+	RequiresExtra(TestQueuedHeader),
+}
+
+#[derive(Clone)]
+struct Target {
+	data: Arc<Mutex<TargetData>>,
+	on_method_call: Arc<dyn Fn(TargetMethod, &mut TargetData) + Send + Sync>,
+}
+
+struct TargetData {
+	best_header_id: Result<TestHeaderId, TestError>,
+	is_known_header_by_hash: HashMap<TestHash, bool>,
+	submitted_headers: HashMap<TestHash, TestQueuedHeader>,
+	submit_headers_result: Option<SubmittedHeaders<TestHeaderId, TestError>>,
+	completed_headers: HashMap<TestHash, TestCompletion>,
+	requires_completion: bool,
+	requires_extra: bool,
+}
+
+impl Target {
+	pub fn new(
+		best_header_id: TestHeaderId,
+		headers: Vec<TestHeaderId>,
+		on_method_call: impl Fn(TargetMethod, &mut TargetData) + Send + Sync + 'static,
+	) -> Self {
+		Target {
+			data: Arc::new(Mutex::new(TargetData {
+				best_header_id: Ok(best_header_id),
+				is_known_header_by_hash: headers.iter().map(|header| (header.1, true)).collect(),
+				submitted_headers: HashMap::new(),
+				submit_headers_result: None,
+				completed_headers: HashMap::new(),
+				requires_completion: false,
+				requires_extra: false,
+			})),
+			on_method_call: Arc::new(on_method_call),
+		}
+	}
+}
+
+#[async_trait]
+impl RelayClient for Target {
+	type Error = TestError;
+
+	async fn reconnect(&mut self) -> Result<(), TestError> {
+		unimplemented!()
+	}
+}
+
+#[async_trait]
+impl TargetClient<TestHeadersSyncPipeline> for Target {
+	async fn best_header_id(&self) -> Result<TestHeaderId, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(TargetMethod::BestHeaderId, &mut *data);
+		data.best_header_id.clone()
+	}
+
+	async fn is_known_header(&self, id: TestHeaderId) -> Result<(TestHeaderId, bool), TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(TargetMethod::IsKnownHeader(id), &mut *data);
+		data.is_known_header_by_hash
+			.get(&id.1)
+			.cloned()
+			.map(|is_known_header| Ok((id, is_known_header)))
+			.unwrap_or(Ok((id, false)))
+	}
+
+	async fn submit_headers(&self, headers: Vec<TestQueuedHeader>) -> SubmittedHeaders<TestHeaderId, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(TargetMethod::SubmitHeaders(headers.clone()), &mut *data);
+		data.submitted_headers
+			.extend(headers.iter().map(|header| (header.id().1, header.clone())));
+		data.submit_headers_result.take().expect("test must accept headers")
+	}
+
+	async fn incomplete_headers_ids(&self) -> Result<HashSet<TestHeaderId>, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(TargetMethod::IncompleteHeadersIds, &mut *data);
+		if data.requires_completion {
+			Ok(data
+				.submitted_headers
+				.iter()
+				.filter(|(hash, _)| !data.completed_headers.contains_key(hash))
+				.map(|(_, header)| header.id())
+				.collect())
+		} else {
+			Ok(HashSet::new())
+		}
+	}
+
+	async fn complete_header(&self, id: TestHeaderId, completion: TestCompletion) -> Result<TestHeaderId, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(TargetMethod::CompleteHeader(id, completion), &mut *data);
+		data.completed_headers.insert(id.1, completion);
+		Ok(id)
+	}
+
+	async fn requires_extra(&self, header: TestQueuedHeader) -> Result<(TestHeaderId, bool), TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(TargetMethod::RequiresExtra(header.clone()), &mut *data);
+		if data.requires_extra {
+			Ok((header.id(), true))
+		} else {
+			Ok((header.id(), false))
+		}
+	}
+}
+
+fn test_tick() -> Duration {
+	// in ideal world that should have been Duration::from_millis(0), because we do not want
+	// to sleep in tests at all, but that could lead to `select! {}` always waking on tick
+	// => not doing actual job
+	Duration::from_millis(10)
+}
+
+fn test_id(number: TestNumber) -> TestHeaderId {
+	HeaderId(number, number)
+}
+
+fn test_header(number: TestNumber) -> TestHeader {
+	let id = test_id(number);
+	TestHeader {
+		hash: id.1,
+		number: id.0,
+		parent_hash: if number == 0 {
+			TestHash::default()
+		} else {
+			test_id(number - 1).1
+		},
+	}
+}
+
+fn test_forked_id(number: TestNumber, forked_from: TestNumber) -> TestHeaderId {
+	const FORK_OFFSET: TestNumber = 1000;
+
+	if number == forked_from {
+		HeaderId(number, number)
+	} else {
+		HeaderId(number, FORK_OFFSET + number)
+	}
+}
+
+fn test_forked_header(number: TestNumber, forked_from: TestNumber) -> TestHeader {
+	let id = test_forked_id(number, forked_from);
+	TestHeader {
+		hash: id.1,
+		number: id.0,
+		parent_hash: if number == 0 {
+			TestHash::default()
+		} else {
+			test_forked_id(number - 1, forked_from).1
+		},
+	}
+}
+
+fn test_completion(id: TestHeaderId) -> TestCompletion {
+	id.0
+}
+
+fn test_extra(id: TestHeaderId) -> TestExtra {
+	id.0
+}
+
+fn source_reject_completion(method: &SourceMethod) {
+	if let SourceMethod::HeaderCompletion(_) = method {
+		unreachable!("HeaderCompletion request is not expected")
+	}
+}
+
+fn source_reject_extra(method: &SourceMethod) {
+	if let SourceMethod::HeaderExtra(_, _) = method {
+		unreachable!("HeaderExtra request is not expected")
+	}
+}
+
+fn target_accept_all_headers(method: &TargetMethod, data: &mut TargetData, requires_extra: bool) {
+	if let TargetMethod::SubmitHeaders(ref submitted) = method {
+		assert_eq!(submitted.iter().all(|header| header.extra().is_some()), requires_extra,);
+
+		data.submit_headers_result = Some(SubmittedHeaders {
+			submitted: submitted.iter().map(|header| header.id()).collect(),
+			..Default::default()
+		});
+	}
+}
+
+fn target_signal_exit_when_header_submitted(
+	method: &TargetMethod,
+	header_id: TestHeaderId,
+	exit_signal: &futures::channel::mpsc::UnboundedSender<()>,
+) {
+	if let TargetMethod::SubmitHeaders(ref submitted) = method {
+		if submitted.iter().any(|header| header.id() == header_id) {
+			exit_signal.unbounded_send(()).unwrap();
+		}
+	}
+}
+
+fn target_signal_exit_when_header_completed(
+	method: &TargetMethod,
+	header_id: TestHeaderId,
+	exit_signal: &futures::channel::mpsc::UnboundedSender<()>,
+) {
+	if let TargetMethod::CompleteHeader(completed_id, _) = method {
+		if *completed_id == header_id {
+			exit_signal.unbounded_send(()).unwrap();
+		}
+	}
+}
+
+fn run_backoff_test(result: Result<(), TestError>) -> (Duration, Duration) {
+	let mut backoff = retry_backoff();
+
+	// no randomness in tests (otherwise intervals may overlap => asserts are failing)
+	backoff.randomization_factor = 0f64;
+
+	// increase backoff's current interval
+	let interval1 = backoff.next_backoff().unwrap();
+	let interval2 = backoff.next_backoff().unwrap();
+	assert!(interval2 > interval1);
+
+	// successful future result leads to backoff's reset
+	let go_offline_future = futures::future::Fuse::terminated();
+	futures::pin_mut!(go_offline_future);
+
+	process_future_result(
+		result,
+		&mut backoff,
+		|_| {},
+		&mut go_offline_future,
+		async_std::task::sleep,
+		|| "Test error".into(),
+	);
+
+	(interval2, backoff.next_backoff().unwrap())
+}
+
+#[test]
+fn process_future_result_resets_backoff_on_success() {
+	let (interval2, interval_after_reset) = run_backoff_test(Ok(()));
+	assert!(interval2 > interval_after_reset);
+}
+
+#[test]
+fn process_future_result_resets_backoff_on_connection_error() {
+	let (interval2, interval_after_reset) = run_backoff_test(Err(TestError(true)));
+	assert!(interval2 > interval_after_reset);
+}
+
+#[test]
+fn process_future_result_does_not_reset_backoff_on_non_connection_error() {
+	let (interval2, interval_after_reset) = run_backoff_test(Err(TestError(false)));
+	assert!(interval2 < interval_after_reset);
+}
+
+struct SyncLoopTestParams {
+	best_source_header: TestHeader,
+	headers_on_source: Vec<(bool, TestHeader)>,
+	best_target_header: TestHeader,
+	headers_on_target: Vec<TestHeader>,
+	target_requires_extra: bool,
+	target_requires_completion: bool,
+	stop_at: TestHeaderId,
+}
+
+fn run_sync_loop_test(params: SyncLoopTestParams) {
+	let (exit_sender, exit_receiver) = futures::channel::mpsc::unbounded();
+	let target_requires_extra = params.target_requires_extra;
+	let target_requires_completion = params.target_requires_completion;
+	let stop_at = params.stop_at;
+	let source = Source::new(
+		params.best_source_header.id(),
+		params.headers_on_source,
+		move |method, _| {
+			if !target_requires_extra {
+				source_reject_extra(&method);
+			}
+			if !target_requires_completion {
+				source_reject_completion(&method);
+			}
+		},
+	);
+	let target = Target::new(
+		params.best_target_header.id(),
+		params.headers_on_target.into_iter().map(|header| header.id()).collect(),
+		move |method, data| {
+			target_accept_all_headers(&method, data, target_requires_extra);
+			if target_requires_completion {
+				target_signal_exit_when_header_completed(&method, stop_at, &exit_sender);
+			} else {
+				target_signal_exit_when_header_submitted(&method, stop_at, &exit_sender);
+			}
+		},
+	);
+	target.data.lock().requires_extra = target_requires_extra;
+	target.data.lock().requires_completion = target_requires_completion;
+
+	run(
+		source,
+		test_tick(),
+		target,
+		test_tick(),
+		(),
+		crate::sync::tests::default_sync_params(),
+		None,
+		exit_receiver.into_future().map(|(_, _)| ()),
+	);
+}
+
+#[test]
+fn sync_loop_is_able_to_synchronize_single_header() {
+	run_sync_loop_test(SyncLoopTestParams {
+		best_source_header: test_header(1),
+		headers_on_source: vec![(true, test_header(1))],
+		best_target_header: test_header(0),
+		headers_on_target: vec![test_header(0)],
+		target_requires_extra: false,
+		target_requires_completion: false,
+		stop_at: test_id(1),
+	});
+}
+
+#[test]
+fn sync_loop_is_able_to_synchronize_single_header_with_extra() {
+	run_sync_loop_test(SyncLoopTestParams {
+		best_source_header: test_header(1),
+		headers_on_source: vec![(true, test_header(1))],
+		best_target_header: test_header(0),
+		headers_on_target: vec![test_header(0)],
+		target_requires_extra: true,
+		target_requires_completion: false,
+		stop_at: test_id(1),
+	});
+}
+
+#[test]
+fn sync_loop_is_able_to_synchronize_single_header_with_completion() {
+	run_sync_loop_test(SyncLoopTestParams {
+		best_source_header: test_header(1),
+		headers_on_source: vec![(true, test_header(1))],
+		best_target_header: test_header(0),
+		headers_on_target: vec![test_header(0)],
+		target_requires_extra: false,
+		target_requires_completion: true,
+		stop_at: test_id(1),
+	});
+}
+
+#[test]
+fn sync_loop_is_able_to_reorganize_from_shorter_fork() {
+	run_sync_loop_test(SyncLoopTestParams {
+		best_source_header: test_header(3),
+		headers_on_source: vec![
+			(true, test_header(1)),
+			(true, test_header(2)),
+			(true, test_header(3)),
+			(false, test_forked_header(1, 0)),
+			(false, test_forked_header(2, 0)),
+		],
+		best_target_header: test_forked_header(2, 0),
+		headers_on_target: vec![test_header(0), test_forked_header(1, 0), test_forked_header(2, 0)],
+		target_requires_extra: false,
+		target_requires_completion: false,
+		stop_at: test_id(3),
+	});
+}
+
+#[test]
+fn sync_loop_is_able_to_reorganize_from_longer_fork() {
+	run_sync_loop_test(SyncLoopTestParams {
+		best_source_header: test_header(3),
+		headers_on_source: vec![
+			(true, test_header(1)),
+			(true, test_header(2)),
+			(true, test_header(3)),
+			(false, test_forked_header(1, 0)),
+			(false, test_forked_header(2, 0)),
+			(false, test_forked_header(3, 0)),
+			(false, test_forked_header(4, 0)),
+			(false, test_forked_header(5, 0)),
+		],
+		best_target_header: test_forked_header(5, 0),
+		headers_on_target: vec![
+			test_header(0),
+			test_forked_header(1, 0),
+			test_forked_header(2, 0),
+			test_forked_header(3, 0),
+			test_forked_header(4, 0),
+			test_forked_header(5, 0),
+		],
+		target_requires_extra: false,
+		target_requires_completion: false,
+		stop_at: test_id(3),
+	});
+}

bridges/relays/generic/headers/src/sync_types.rs

@@ -0,0 +1,189 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Types that are used by headers synchronization components.
+
+use relay_utils::{format_ids, HeaderId};
+use std::{ops::Deref, sync::Arc};
+
+/// Ethereum header synchronization status.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum HeaderStatus {
+	/// Header is unknown.
+	Unknown,
+	/// Header is in MaybeOrphan queue.
+	MaybeOrphan,
+	/// Header is in Orphan queue.
+	Orphan,
+	/// Header is in MaybeExtra queue.
+	MaybeExtra,
+	/// Header is in Extra queue.
+	Extra,
+	/// Header is in Ready queue.
+	Ready,
+	/// Header is in Incomplete queue.
+	Incomplete,
+	/// Header has been recently submitted to the target node.
+	Submitted,
+	/// Header is known to the target node.
+	Synced,
+}
+
+/// Headers synchronization pipeline.
+pub trait HeadersSyncPipeline: Clone + Send + Sync {
+	/// Name of the headers source.
+	const SOURCE_NAME: &'static str;
+	/// Name of the headers target.
+	const TARGET_NAME: &'static str;
+
+	/// Headers we're syncing are identified by this hash.
+	type Hash: Eq + Clone + Copy + Send + Sync + std::fmt::Debug + std::fmt::Display + std::hash::Hash;
+	/// Headers we're syncing are identified by this number.
+	type Number: relay_utils::BlockNumberBase;
+	/// Type of header that we're syncing.
+	type Header: SourceHeader<Self::Hash, Self::Number>;
+	/// Type of extra data for the header that we're receiving from the source node:
+	/// 1) extra data is required for some headers;
+	/// 2) target node may answer if it'll require extra data before header is submitted;
+	/// 3) extra data available since the header creation time;
+	/// 4) header and extra data are submitted in single transaction.
+	///
+	/// Example: Ethereum transactions receipts.
+	type Extra: Clone + Send + Sync + PartialEq + std::fmt::Debug;
+	/// Type of data required to 'complete' header that we're receiving from the source node:
+	/// 1) completion data is required for some headers;
+	/// 2) target node can't answer if it'll require completion data before header is accepted;
+	/// 3) completion data may be generated after header generation;
+	/// 4) header and completion data are submitted in separate transactions.
+	///
+	/// Example: Substrate GRANDPA justifications.
+	type Completion: Clone + Send + Sync + std::fmt::Debug;
+
+	/// Function used to estimate size of target-encoded header.
+	fn estimate_size(source: &QueuedHeader<Self>) -> usize;
+}
+
+/// A HeaderId for `HeaderSyncPipeline`.
+pub type HeaderIdOf<P> = HeaderId<<P as HeadersSyncPipeline>::Hash, <P as HeadersSyncPipeline>::Number>;
+
+/// Header that we're receiving from source node.
+pub trait SourceHeader<Hash, Number>: Clone + std::fmt::Debug + PartialEq + Send + Sync {
+	/// Returns ID of header.
+	fn id(&self) -> HeaderId<Hash, Number>;
+	/// Returns ID of parent header.
+	///
+	/// Panics if called for genesis header.
+	fn parent_id(&self) -> HeaderId<Hash, Number>;
+}
+
+/// Header how it's stored in the synchronization queue.
+#[derive(Clone, Debug, PartialEq)]
+pub struct QueuedHeader<P: HeadersSyncPipeline>(Arc<QueuedHeaderData<P>>);
+
+impl<P: HeadersSyncPipeline> QueuedHeader<P> {
+	/// Creates new queued header.
+	pub fn new(header: P::Header) -> Self {
+		QueuedHeader(Arc::new(QueuedHeaderData { header, extra: None }))
+	}
+
+	/// Set associated extra data.
+	pub fn set_extra(self, extra: P::Extra) -> Self {
+		QueuedHeader(Arc::new(QueuedHeaderData {
+			header: Arc::try_unwrap(self.0)
+				.map(|data| data.header)
+				.unwrap_or_else(|data| data.header.clone()),
+			extra: Some(extra),
+		}))
+	}
+}
+
+impl<P: HeadersSyncPipeline> Deref for QueuedHeader<P> {
+	type Target = QueuedHeaderData<P>;
+
+	fn deref(&self) -> &Self::Target {
+		&self.0
+	}
+}
+
+/// Header how it's stored in the synchronization queue.
+#[derive(Clone, Debug, Default, PartialEq)]
+pub struct QueuedHeaderData<P: HeadersSyncPipeline> {
+	header: P::Header,
+	extra: Option<P::Extra>,
+}
+
+impl<P: HeadersSyncPipeline> QueuedHeader<P> {
+	/// Returns ID of header.
+	pub fn id(&self) -> HeaderId<P::Hash, P::Number> {
+		self.header.id()
+	}
+
+	/// Returns ID of parent header.
+	pub fn parent_id(&self) -> HeaderId<P::Hash, P::Number> {
+		self.header.parent_id()
+	}
+
+	/// Returns reference to header.
+	pub fn header(&self) -> &P::Header {
+		&self.header
+	}
+
+	/// Returns reference to associated extra data.
+	pub fn extra(&self) -> &Option<P::Extra> {
+		&self.extra
+	}
+}
+
+/// Headers submission result.
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+pub struct SubmittedHeaders<Id, Error> {
+	/// IDs of headers that have been submitted to target node.
+	pub submitted: Vec<Id>,
+	/// IDs of incomplete headers. These headers were submitted (so this id is also in `submitted` vec),
+	/// but all descendants are not.
+	pub incomplete: Vec<Id>,
+	/// IDs of ignored headers that we have decided not to submit (they're either rejected by
+	/// target node immediately, or they're descendants of incomplete headers).
+	pub rejected: Vec<Id>,
+	/// Fatal target node error, if it has occured during submission.
+	pub fatal_error: Option<Error>,
+}
+
+impl<Id, Error> Default for SubmittedHeaders<Id, Error> {
+	fn default() -> Self {
+		SubmittedHeaders {
+			submitted: Vec::new(),
+			incomplete: Vec::new(),
+			rejected: Vec::new(),
+			fatal_error: None,
+		}
+	}
+}
+
+impl<Id: std::fmt::Debug, Error> std::fmt::Display for SubmittedHeaders<Id, Error> {
+	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+		let submitted = format_ids(self.submitted.iter());
+		let incomplete = format_ids(self.incomplete.iter());
+		let rejected = format_ids(self.rejected.iter());
+
+		write!(
+			f,
+			"Submitted: {}, Incomplete: {}, Rejected: {}",
+			submitted, incomplete, rejected
+		)
+	}
+}

bridges/relays/generic/messages/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "messages-relay"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+
+[dependencies]
+async-std = "1.6.5"
+async-trait = "0.1.40"
+futures = "0.3.5"
+hex = "0.4"
+log = "0.4.11"
+parking_lot = "0.11.0"
+
+# Bridge Dependencies
+
+bp-message-lane = { path = "../../../primitives/message-lane" }
+relay-utils = { path = "../utils" }

bridges/relays/generic/messages/src/lib.rs

@@ -0,0 +1,36 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Relaying [`message-lane`](../pallet_message_lane/index.html) application specific
+//! data. Message lane allows sending arbitrary messages between bridged chains. This
+//! module provides entrypoint that starts reading messages from given message lane
+//! of source chain and submits proof-of-message-at-source-chain transactions to the
+//! target chain. Additionaly, proofs-of-messages-delivery are sent back from the
+//! target chain to the source chain.
+
+// required for futures::select!
+#![recursion_limit = "1024"]
+#![warn(missing_docs)]
+
+mod metrics;
+
+pub mod message_lane;
+pub mod message_lane_loop;
+
+mod message_race_delivery;
+mod message_race_loop;
+mod message_race_receiving;
+mod message_race_strategy;

bridges/relays/generic/messages/src/message_lane.rs

@@ -0,0 +1,52 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! One-way message lane types. Within single one-way lane we have three 'races' where we try to:
+//!
+//! 1) relay new messages from source to target node;
+//! 2) relay proof-of-delivery from target to source node.
+
+use relay_utils::{BlockNumberBase, HeaderId};
+use std::fmt::Debug;
+
+/// One-way message lane.
+pub trait MessageLane: Clone + Send + Sync {
+	/// Name of the messages source.
+	const SOURCE_NAME: &'static str;
+	/// Name of the messages target.
+	const TARGET_NAME: &'static str;
+
+	/// Messages proof.
+	type MessagesProof: Clone + Debug + Send + Sync;
+	/// Messages receiving proof.
+	type MessagesReceivingProof: Clone + Debug + Send + Sync;
+
+	/// Number of the source header.
+	type SourceHeaderNumber: BlockNumberBase;
+	/// Hash of the source header.
+	type SourceHeaderHash: Clone + Debug + Default + PartialEq + Send + Sync;
+
+	/// Number of the target header.
+	type TargetHeaderNumber: BlockNumberBase;
+	/// Hash of the target header.
+	type TargetHeaderHash: Clone + Debug + Default + PartialEq + Send + Sync;
+}
+
+/// Source header id within given one-way message lane.
+pub type SourceHeaderIdOf<P> = HeaderId<<P as MessageLane>::SourceHeaderHash, <P as MessageLane>::SourceHeaderNumber>;
+
+/// Target header id within given one-way message lane.
+pub type TargetHeaderIdOf<P> = HeaderId<<P as MessageLane>::TargetHeaderHash, <P as MessageLane>::TargetHeaderNumber>;

bridges/relays/generic/messages/src/message_lane_loop.rs

@@ -0,0 +1,841 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Message delivery loop. Designed to work with message-lane pallet.
+//!
+//! Single relay instance delivers messages of single lane in single direction.
+//! To serve two-way lane, you would need two instances of relay.
+//! To serve N two-way lanes, you would need N*2 instances of relay.
+//!
+//! Please keep in mind that the best header in this file is actually best
+//! finalized header. I.e. when talking about headers in lane context, we
+//! only care about finalized headers.
+
+use crate::message_lane::{MessageLane, SourceHeaderIdOf, TargetHeaderIdOf};
+use crate::message_race_delivery::run as run_message_delivery_race;
+use crate::message_race_receiving::run as run_message_receiving_race;
+use crate::metrics::MessageLaneLoopMetrics;
+
+use async_trait::async_trait;
+use bp_message_lane::{LaneId, MessageNonce, UnrewardedRelayersState, Weight};
+use futures::{channel::mpsc::unbounded, future::FutureExt, stream::StreamExt};
+use relay_utils::{
+	interval,
+	metrics::{start as metrics_start, GlobalMetrics, MetricsParams},
+	process_future_result,
+	relay_loop::Client as RelayClient,
+	retry_backoff, FailedClient,
+};
+use std::{collections::BTreeMap, fmt::Debug, future::Future, ops::RangeInclusive, time::Duration};
+
+/// Message lane loop configuration params.
+#[derive(Debug, Clone)]
+pub struct Params {
+	/// Id of lane this loop is servicing.
+	pub lane: LaneId,
+	/// Interval at which we ask target node about its updates.
+	pub source_tick: Duration,
+	/// Interval at which we ask target node about its updates.
+	pub target_tick: Duration,
+	/// Delay between moments when connection error happens and our reconnect attempt.
+	pub reconnect_delay: Duration,
+	/// The loop will auto-restart if there has been no updates during this period.
+	pub stall_timeout: Duration,
+	/// Message delivery race parameters.
+	pub delivery_params: MessageDeliveryParams,
+}
+
+/// Message delivery race parameters.
+#[derive(Debug, Clone)]
+pub struct MessageDeliveryParams {
+	/// Maximal number of unconfirmed relayer entries at the inbound lane. If there's that number of entries
+	/// in the `InboundLaneData::relayers` set, all new messages will be rejected until reward payment will
+	/// be proved (by including outbound lane state to the message delivery transaction).
+	pub max_unrewarded_relayer_entries_at_target: MessageNonce,
+	/// Message delivery race will stop delivering messages if there are `max_unconfirmed_nonces_at_target`
+	/// unconfirmed nonces on the target node. The race would continue once they're confirmed by the
+	/// receiving race.
+	pub max_unconfirmed_nonces_at_target: MessageNonce,
+	/// Maximal number of relayed messages in single delivery transaction.
+	pub max_messages_in_single_batch: MessageNonce,
+	/// Maximal cumulative dispatch weight of relayed messages in single delivery transaction.
+	pub max_messages_weight_in_single_batch: Weight,
+	/// Maximal cumulative size of relayed messages in single delivery transaction.
+	pub max_messages_size_in_single_batch: usize,
+}
+
+/// Message weights.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct MessageWeights {
+	/// Message dispatch weight.
+	pub weight: Weight,
+	/// Message size (number of bytes in encoded payload).
+	pub size: usize,
+}
+
+/// Messages weights map.
+pub type MessageWeightsMap = BTreeMap<MessageNonce, MessageWeights>;
+
+/// Message delivery race proof parameters.
+#[derive(Debug, PartialEq)]
+pub struct MessageProofParameters {
+	/// Include outbound lane state proof?
+	pub outbound_state_proof_required: bool,
+	/// Cumulative dispatch weight of messages that we're building proof for.
+	pub dispatch_weight: Weight,
+}
+
+/// Source client trait.
+#[async_trait]
+pub trait SourceClient<P: MessageLane>: RelayClient {
+	/// Returns state of the client.
+	async fn state(&self) -> Result<SourceClientState<P>, Self::Error>;
+
+	/// Get nonce of instance of latest generated message.
+	async fn latest_generated_nonce(
+		&self,
+		id: SourceHeaderIdOf<P>,
+	) -> Result<(SourceHeaderIdOf<P>, MessageNonce), Self::Error>;
+	/// Get nonce of the latest message, which receiving has been confirmed by the target chain.
+	async fn latest_confirmed_received_nonce(
+		&self,
+		id: SourceHeaderIdOf<P>,
+	) -> Result<(SourceHeaderIdOf<P>, MessageNonce), Self::Error>;
+
+	/// Returns mapping of message nonces, generated on this client, to their weights.
+	///
+	/// Some weights may be missing from returned map, if corresponding messages were pruned at
+	/// the source chain.
+	async fn generated_messages_weights(
+		&self,
+		id: SourceHeaderIdOf<P>,
+		nonces: RangeInclusive<MessageNonce>,
+	) -> Result<MessageWeightsMap, Self::Error>;
+
+	/// Prove messages in inclusive range [begin; end].
+	async fn prove_messages(
+		&self,
+		id: SourceHeaderIdOf<P>,
+		nonces: RangeInclusive<MessageNonce>,
+		proof_parameters: MessageProofParameters,
+	) -> Result<(SourceHeaderIdOf<P>, RangeInclusive<MessageNonce>, P::MessagesProof), Self::Error>;
+
+	/// Submit messages receiving proof.
+	async fn submit_messages_receiving_proof(
+		&self,
+		generated_at_block: TargetHeaderIdOf<P>,
+		proof: P::MessagesReceivingProof,
+	) -> Result<(), Self::Error>;
+}
+
+/// Target client trait.
+#[async_trait]
+pub trait TargetClient<P: MessageLane>: RelayClient {
+	/// Returns state of the client.
+	async fn state(&self) -> Result<TargetClientState<P>, Self::Error>;
+
+	/// Get nonce of latest received message.
+	async fn latest_received_nonce(
+		&self,
+		id: TargetHeaderIdOf<P>,
+	) -> Result<(TargetHeaderIdOf<P>, MessageNonce), Self::Error>;
+
+	/// Get nonce of latest confirmed message.
+	async fn latest_confirmed_received_nonce(
+		&self,
+		id: TargetHeaderIdOf<P>,
+	) -> Result<(TargetHeaderIdOf<P>, MessageNonce), Self::Error>;
+	/// Get state of unrewarded relayers set at the inbound lane.
+	async fn unrewarded_relayers_state(
+		&self,
+		id: TargetHeaderIdOf<P>,
+	) -> Result<(TargetHeaderIdOf<P>, UnrewardedRelayersState), Self::Error>;
+
+	/// Prove messages receiving at given block.
+	async fn prove_messages_receiving(
+		&self,
+		id: TargetHeaderIdOf<P>,
+	) -> Result<(TargetHeaderIdOf<P>, P::MessagesReceivingProof), Self::Error>;
+
+	/// Submit messages proof.
+	async fn submit_messages_proof(
+		&self,
+		generated_at_header: SourceHeaderIdOf<P>,
+		nonces: RangeInclusive<MessageNonce>,
+		proof: P::MessagesProof,
+	) -> Result<RangeInclusive<MessageNonce>, Self::Error>;
+}
+
+/// State of the client.
+#[derive(Clone, Debug, Default, PartialEq)]
+pub struct ClientState<SelfHeaderId, PeerHeaderId> {
+	/// Best header id of this chain.
+	pub best_self: SelfHeaderId,
+	/// Best finalized header id of this chain.
+	pub best_finalized_self: SelfHeaderId,
+	/// Best finalized header id of the peer chain read at the best block of this chain (at `best_finalized_self`).
+	pub best_finalized_peer_at_best_self: PeerHeaderId,
+}
+
+/// State of source client in one-way message lane.
+pub type SourceClientState<P> = ClientState<SourceHeaderIdOf<P>, TargetHeaderIdOf<P>>;
+
+/// State of target client in one-way message lane.
+pub type TargetClientState<P> = ClientState<TargetHeaderIdOf<P>, SourceHeaderIdOf<P>>;
+
+/// Both clients state.
+#[derive(Debug, Default)]
+pub struct ClientsState<P: MessageLane> {
+	/// Source client state.
+	pub source: Option<SourceClientState<P>>,
+	/// Target client state.
+	pub target: Option<TargetClientState<P>>,
+}
+
+/// Run message lane service loop.
+pub fn run<P: MessageLane>(
+	params: Params,
+	source_client: impl SourceClient<P>,
+	target_client: impl TargetClient<P>,
+	metrics_params: Option<MetricsParams>,
+	exit_signal: impl Future<Output = ()>,
+) {
+	let exit_signal = exit_signal.shared();
+	let metrics_global = GlobalMetrics::default();
+	let metrics_msg = MessageLaneLoopMetrics::default();
+	let metrics_enabled = metrics_params.is_some();
+	metrics_start(
+		format!(
+			"{}_to_{}_MessageLane_{}",
+			P::SOURCE_NAME,
+			P::TARGET_NAME,
+			hex::encode(params.lane)
+		),
+		metrics_params,
+		&metrics_global,
+		&metrics_msg,
+	);
+
+	relay_utils::relay_loop::run(
+		params.reconnect_delay,
+		source_client,
+		target_client,
+		|source_client, target_client| {
+			run_until_connection_lost(
+				params.clone(),
+				source_client,
+				target_client,
+				if metrics_enabled {
+					Some(metrics_global.clone())
+				} else {
+					None
+				},
+				if metrics_enabled {
+					Some(metrics_msg.clone())
+				} else {
+					None
+				},
+				exit_signal.clone(),
+			)
+		},
+	);
+}
+
+/// Run one-way message delivery loop until connection with target or source node is lost, or exit signal is received.
+async fn run_until_connection_lost<P: MessageLane, SC: SourceClient<P>, TC: TargetClient<P>>(
+	params: Params,
+	source_client: SC,
+	target_client: TC,
+	metrics_global: Option<GlobalMetrics>,
+	metrics_msg: Option<MessageLaneLoopMetrics>,
+	exit_signal: impl Future<Output = ()>,
+) -> Result<(), FailedClient> {
+	let mut source_retry_backoff = retry_backoff();
+	let mut source_client_is_online = false;
+	let mut source_state_required = true;
+	let source_state = source_client.state().fuse();
+	let source_go_offline_future = futures::future::Fuse::terminated();
+	let source_tick_stream = interval(params.source_tick).fuse();
+
+	let mut target_retry_backoff = retry_backoff();
+	let mut target_client_is_online = false;
+	let mut target_state_required = true;
+	let target_state = target_client.state().fuse();
+	let target_go_offline_future = futures::future::Fuse::terminated();
+	let target_tick_stream = interval(params.target_tick).fuse();
+
+	let (
+		(delivery_source_state_sender, delivery_source_state_receiver),
+		(delivery_target_state_sender, delivery_target_state_receiver),
+	) = (unbounded(), unbounded());
+	let delivery_race_loop = run_message_delivery_race(
+		source_client.clone(),
+		delivery_source_state_receiver,
+		target_client.clone(),
+		delivery_target_state_receiver,
+		params.stall_timeout,
+		metrics_msg.clone(),
+		params.delivery_params,
+	)
+	.fuse();
+
+	let (
+		(receiving_source_state_sender, receiving_source_state_receiver),
+		(receiving_target_state_sender, receiving_target_state_receiver),
+	) = (unbounded(), unbounded());
+	let receiving_race_loop = run_message_receiving_race(
+		source_client.clone(),
+		receiving_source_state_receiver,
+		target_client.clone(),
+		receiving_target_state_receiver,
+		params.stall_timeout,
+		metrics_msg.clone(),
+	)
+	.fuse();
+
+	let exit_signal = exit_signal.fuse();
+
+	futures::pin_mut!(
+		source_state,
+		source_go_offline_future,
+		source_tick_stream,
+		target_state,
+		target_go_offline_future,
+		target_tick_stream,
+		delivery_race_loop,
+		receiving_race_loop,
+		exit_signal
+	);
+
+	loop {
+		futures::select! {
+			new_source_state = source_state => {
+				source_state_required = false;
+
+				source_client_is_online = process_future_result(
+					new_source_state,
+					&mut source_retry_backoff,
+					|new_source_state| {
+						log::debug!(
+							target: "bridge",
+							"Received state from {} node: {:?}",
+							P::SOURCE_NAME,
+							new_source_state,
+						);
+						let _ = delivery_source_state_sender.unbounded_send(new_source_state.clone());
+						let _ = receiving_source_state_sender.unbounded_send(new_source_state.clone());
+
+						if let Some(metrics_msg) = metrics_msg.as_ref() {
+							metrics_msg.update_source_state::<P>(new_source_state);
+						}
+					},
+					&mut source_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving state from {} node", P::SOURCE_NAME),
+				).fail_if_connection_error(FailedClient::Source)?;
+			},
+			_ = source_go_offline_future => {
+				source_client_is_online = true;
+			},
+			_ = source_tick_stream.next() => {
+				source_state_required = true;
+			},
+			new_target_state = target_state => {
+				target_state_required = false;
+
+				target_client_is_online = process_future_result(
+					new_target_state,
+					&mut target_retry_backoff,
+					|new_target_state| {
+						log::debug!(
+							target: "bridge",
+							"Received state from {} node: {:?}",
+							P::TARGET_NAME,
+							new_target_state,
+						);
+						let _ = delivery_target_state_sender.unbounded_send(new_target_state.clone());
+						let _ = receiving_target_state_sender.unbounded_send(new_target_state.clone());
+
+						if let Some(metrics_msg) = metrics_msg.as_ref() {
+							metrics_msg.update_target_state::<P>(new_target_state);
+						}
+					},
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving state from {} node", P::TARGET_NAME),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+			_ = target_go_offline_future => {
+				target_client_is_online = true;
+			},
+			_ = target_tick_stream.next() => {
+				target_state_required = true;
+			},
+
+			delivery_error = delivery_race_loop => {
+				match delivery_error {
+					Ok(_) => unreachable!("only ends with error; qed"),
+					Err(err) => return Err(err),
+				}
+			},
+			receiving_error = receiving_race_loop => {
+				match receiving_error {
+					Ok(_) => unreachable!("only ends with error; qed"),
+					Err(err) => return Err(err),
+				}
+			},
+
+			() = exit_signal => {
+				return Ok(());
+			}
+		}
+
+		if let Some(ref metrics_global) = metrics_global {
+			metrics_global.update().await;
+		}
+
+		if source_client_is_online && source_state_required {
+			log::debug!(target: "bridge", "Asking {} node about its state", P::SOURCE_NAME);
+			source_state.set(source_client.state().fuse());
+			source_client_is_online = false;
+		}
+
+		if target_client_is_online && target_state_required {
+			log::debug!(target: "bridge", "Asking {} node about its state", P::TARGET_NAME);
+			target_state.set(target_client.state().fuse());
+			target_client_is_online = false;
+		}
+	}
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+	use super::*;
+	use futures::stream::StreamExt;
+	use parking_lot::Mutex;
+	use relay_utils::{HeaderId, MaybeConnectionError};
+	use std::sync::Arc;
+
+	pub fn header_id(number: TestSourceHeaderNumber) -> TestSourceHeaderId {
+		HeaderId(number, number)
+	}
+
+	pub type TestSourceHeaderId = HeaderId<TestSourceHeaderNumber, TestSourceHeaderHash>;
+	pub type TestTargetHeaderId = HeaderId<TestTargetHeaderNumber, TestTargetHeaderHash>;
+
+	pub type TestMessagesProof = (RangeInclusive<MessageNonce>, Option<MessageNonce>);
+	pub type TestMessagesReceivingProof = MessageNonce;
+
+	pub type TestSourceHeaderNumber = u64;
+	pub type TestSourceHeaderHash = u64;
+
+	pub type TestTargetHeaderNumber = u64;
+	pub type TestTargetHeaderHash = u64;
+
+	#[derive(Debug)]
+	pub struct TestError;
+
+	impl MaybeConnectionError for TestError {
+		fn is_connection_error(&self) -> bool {
+			true
+		}
+	}
+
+	#[derive(Clone)]
+	pub struct TestMessageLane;
+
+	impl MessageLane for TestMessageLane {
+		const SOURCE_NAME: &'static str = "TestSource";
+		const TARGET_NAME: &'static str = "TestTarget";
+
+		type MessagesProof = TestMessagesProof;
+		type MessagesReceivingProof = TestMessagesReceivingProof;
+
+		type SourceHeaderNumber = TestSourceHeaderNumber;
+		type SourceHeaderHash = TestSourceHeaderHash;
+
+		type TargetHeaderNumber = TestTargetHeaderNumber;
+		type TargetHeaderHash = TestTargetHeaderHash;
+	}
+
+	#[derive(Debug, Default, Clone)]
+	pub struct TestClientData {
+		is_source_fails: bool,
+		is_source_reconnected: bool,
+		source_state: SourceClientState<TestMessageLane>,
+		source_latest_generated_nonce: MessageNonce,
+		source_latest_confirmed_received_nonce: MessageNonce,
+		submitted_messages_receiving_proofs: Vec<TestMessagesReceivingProof>,
+		is_target_fails: bool,
+		is_target_reconnected: bool,
+		target_state: SourceClientState<TestMessageLane>,
+		target_latest_received_nonce: MessageNonce,
+		target_latest_confirmed_received_nonce: MessageNonce,
+		submitted_messages_proofs: Vec<TestMessagesProof>,
+	}
+
+	#[derive(Clone)]
+	pub struct TestSourceClient {
+		data: Arc<Mutex<TestClientData>>,
+		tick: Arc<dyn Fn(&mut TestClientData) + Send + Sync>,
+	}
+
+	#[async_trait]
+	impl RelayClient for TestSourceClient {
+		type Error = TestError;
+
+		async fn reconnect(&mut self) -> Result<(), TestError> {
+			{
+				let mut data = self.data.lock();
+				(self.tick)(&mut *data);
+				data.is_source_reconnected = true;
+			}
+			Ok(())
+		}
+	}
+
+	#[async_trait]
+	impl SourceClient<TestMessageLane> for TestSourceClient {
+		async fn state(&self) -> Result<SourceClientState<TestMessageLane>, TestError> {
+			let mut data = self.data.lock();
+			(self.tick)(&mut *data);
+			if data.is_source_fails {
+				return Err(TestError);
+			}
+			Ok(data.source_state.clone())
+		}
+
+		async fn latest_generated_nonce(
+			&self,
+			id: SourceHeaderIdOf<TestMessageLane>,
+		) -> Result<(SourceHeaderIdOf<TestMessageLane>, MessageNonce), TestError> {
+			let mut data = self.data.lock();
+			(self.tick)(&mut *data);
+			if data.is_source_fails {
+				return Err(TestError);
+			}
+			Ok((id, data.source_latest_generated_nonce))
+		}
+
+		async fn latest_confirmed_received_nonce(
+			&self,
+			id: SourceHeaderIdOf<TestMessageLane>,
+		) -> Result<(SourceHeaderIdOf<TestMessageLane>, MessageNonce), TestError> {
+			let mut data = self.data.lock();
+			(self.tick)(&mut *data);
+			Ok((id, data.source_latest_confirmed_received_nonce))
+		}
+
+		async fn generated_messages_weights(
+			&self,
+			_id: SourceHeaderIdOf<TestMessageLane>,
+			nonces: RangeInclusive<MessageNonce>,
+		) -> Result<MessageWeightsMap, TestError> {
+			Ok(nonces
+				.map(|nonce| (nonce, MessageWeights { weight: 1, size: 1 }))
+				.collect())
+		}
+
+		async fn prove_messages(
+			&self,
+			id: SourceHeaderIdOf<TestMessageLane>,
+			nonces: RangeInclusive<MessageNonce>,
+			proof_parameters: MessageProofParameters,
+		) -> Result<
+			(
+				SourceHeaderIdOf<TestMessageLane>,
+				RangeInclusive<MessageNonce>,
+				TestMessagesProof,
+			),
+			TestError,
+		> {
+			let mut data = self.data.lock();
+			(self.tick)(&mut *data);
+			Ok((
+				id,
+				nonces.clone(),
+				(
+					nonces,
+					if proof_parameters.outbound_state_proof_required {
+						Some(data.source_latest_confirmed_received_nonce)
+					} else {
+						None
+					},
+				),
+			))
+		}
+
+		async fn submit_messages_receiving_proof(
+			&self,
+			_generated_at_block: TargetHeaderIdOf<TestMessageLane>,
+			proof: TestMessagesReceivingProof,
+		) -> Result<(), TestError> {
+			let mut data = self.data.lock();
+			(self.tick)(&mut *data);
+			data.submitted_messages_receiving_proofs.push(proof);
+			data.source_latest_confirmed_received_nonce = proof;
+			Ok(())
+		}
+	}
+
+	#[derive(Clone)]
+	pub struct TestTargetClient {
+		data: Arc<Mutex<TestClientData>>,
+		tick: Arc<dyn Fn(&mut TestClientData) + Send + Sync>,
+	}
+
+	#[async_trait]
+	impl RelayClient for TestTargetClient {
+		type Error = TestError;
+
+		async fn reconnect(&mut self) -> Result<(), TestError> {
+			{
+				let mut data = self.data.lock();
+				(self.tick)(&mut *data);
+				data.is_target_reconnected = true;
+			}
+			Ok(())
+		}
+	}
+
+	#[async_trait]
+	impl TargetClient<TestMessageLane> for TestTargetClient {
+		async fn state(&self) -> Result<TargetClientState<TestMessageLane>, TestError> {
+			let mut data = self.data.lock();
+			(self.tick)(&mut *data);
+			if data.is_target_fails {
+				return Err(TestError);
+			}
+			Ok(data.target_state.clone())
+		}
+
+		async fn latest_received_nonce(
+			&self,
+			id: TargetHeaderIdOf<TestMessageLane>,
+		) -> Result<(TargetHeaderIdOf<TestMessageLane>, MessageNonce), TestError> {
+			let mut data = self.data.lock();
+			(self.tick)(&mut *data);
+			if data.is_target_fails {
+				return Err(TestError);
+			}
+			Ok((id, data.target_latest_received_nonce))
+		}
+
+		async fn unrewarded_relayers_state(
+			&self,
+			id: TargetHeaderIdOf<TestMessageLane>,
+		) -> Result<(TargetHeaderIdOf<TestMessageLane>, UnrewardedRelayersState), TestError> {
+			Ok((
+				id,
+				UnrewardedRelayersState {
+					unrewarded_relayer_entries: 0,
+					messages_in_oldest_entry: 0,
+					total_messages: 0,
+				},
+			))
+		}
+
+		async fn latest_confirmed_received_nonce(
+			&self,
+			id: TargetHeaderIdOf<TestMessageLane>,
+		) -> Result<(TargetHeaderIdOf<TestMessageLane>, MessageNonce), TestError> {
+			let mut data = self.data.lock();
+			(self.tick)(&mut *data);
+			if data.is_target_fails {
+				return Err(TestError);
+			}
+			Ok((id, data.target_latest_confirmed_received_nonce))
+		}
+
+		async fn prove_messages_receiving(
+			&self,
+			id: TargetHeaderIdOf<TestMessageLane>,
+		) -> Result<(TargetHeaderIdOf<TestMessageLane>, TestMessagesReceivingProof), TestError> {
+			Ok((id, self.data.lock().target_latest_received_nonce))
+		}
+
+		async fn submit_messages_proof(
+			&self,
+			_generated_at_header: SourceHeaderIdOf<TestMessageLane>,
+			nonces: RangeInclusive<MessageNonce>,
+			proof: TestMessagesProof,
+		) -> Result<RangeInclusive<MessageNonce>, TestError> {
+			let mut data = self.data.lock();
+			(self.tick)(&mut *data);
+			if data.is_target_fails {
+				return Err(TestError);
+			}
+			data.target_state.best_self =
+				HeaderId(data.target_state.best_self.0 + 1, data.target_state.best_self.1 + 1);
+			data.target_latest_received_nonce = *proof.0.end();
+			if let Some(target_latest_confirmed_received_nonce) = proof.1 {
+				data.target_latest_confirmed_received_nonce = target_latest_confirmed_received_nonce;
+			}
+			data.submitted_messages_proofs.push(proof);
+			Ok(nonces)
+		}
+	}
+
+	fn run_loop_test(
+		data: TestClientData,
+		source_tick: Arc<dyn Fn(&mut TestClientData) + Send + Sync>,
+		target_tick: Arc<dyn Fn(&mut TestClientData) + Send + Sync>,
+		exit_signal: impl Future<Output = ()>,
+	) -> TestClientData {
+		async_std::task::block_on(async {
+			let data = Arc::new(Mutex::new(data));
+
+			let source_client = TestSourceClient {
+				data: data.clone(),
+				tick: source_tick,
+			};
+			let target_client = TestTargetClient {
+				data: data.clone(),
+				tick: target_tick,
+			};
+			run(
+				Params {
+					lane: [0, 0, 0, 0],
+					source_tick: Duration::from_millis(100),
+					target_tick: Duration::from_millis(100),
+					reconnect_delay: Duration::from_millis(0),
+					stall_timeout: Duration::from_millis(60 * 1000),
+					delivery_params: MessageDeliveryParams {
+						max_unrewarded_relayer_entries_at_target: 4,
+						max_unconfirmed_nonces_at_target: 4,
+						max_messages_in_single_batch: 4,
+						max_messages_weight_in_single_batch: 4,
+						max_messages_size_in_single_batch: 4,
+					},
+				},
+				source_client,
+				target_client,
+				None,
+				exit_signal,
+			);
+			let result = data.lock().clone();
+			result
+		})
+	}
+
+	#[test]
+	fn message_lane_loop_is_able_to_recover_from_connection_errors() {
+		// with this configuration, source client will return Err, making source client
+		// reconnect. Then the target client will fail with Err + reconnect. Then we finally
+		// able to deliver messages.
+		let (exit_sender, exit_receiver) = unbounded();
+		let result = run_loop_test(
+			TestClientData {
+				is_source_fails: true,
+				source_state: ClientState {
+					best_self: HeaderId(0, 0),
+					best_finalized_self: HeaderId(0, 0),
+					best_finalized_peer_at_best_self: HeaderId(0, 0),
+				},
+				source_latest_generated_nonce: 1,
+				target_state: ClientState {
+					best_self: HeaderId(0, 0),
+					best_finalized_self: HeaderId(0, 0),
+					best_finalized_peer_at_best_self: HeaderId(0, 0),
+				},
+				target_latest_received_nonce: 0,
+				..Default::default()
+			},
+			Arc::new(|data: &mut TestClientData| {
+				if data.is_source_reconnected {
+					data.is_source_fails = false;
+					data.is_target_fails = true;
+				}
+			}),
+			Arc::new(move |data: &mut TestClientData| {
+				if data.is_target_reconnected {
+					data.is_target_fails = false;
+				}
+				if data.target_state.best_finalized_peer_at_best_self.0 < 10 {
+					data.target_state.best_finalized_peer_at_best_self = HeaderId(
+						data.target_state.best_finalized_peer_at_best_self.0 + 1,
+						data.target_state.best_finalized_peer_at_best_self.0 + 1,
+					);
+				}
+				if !data.submitted_messages_proofs.is_empty() {
+					exit_sender.unbounded_send(()).unwrap();
+				}
+			}),
+			exit_receiver.into_future().map(|(_, _)| ()),
+		);
+
+		assert_eq!(result.submitted_messages_proofs, vec![(1..=1, None)],);
+	}
+
+	#[test]
+	fn message_lane_loop_works() {
+		let (exit_sender, exit_receiver) = unbounded();
+		let result = run_loop_test(
+			TestClientData {
+				source_state: ClientState {
+					best_self: HeaderId(10, 10),
+					best_finalized_self: HeaderId(10, 10),
+					best_finalized_peer_at_best_self: HeaderId(0, 0),
+				},
+				source_latest_generated_nonce: 10,
+				target_state: ClientState {
+					best_self: HeaderId(0, 0),
+					best_finalized_self: HeaderId(0, 0),
+					best_finalized_peer_at_best_self: HeaderId(0, 0),
+				},
+				target_latest_received_nonce: 0,
+				..Default::default()
+			},
+			Arc::new(|_: &mut TestClientData| {}),
+			Arc::new(move |data: &mut TestClientData| {
+				// syncing source headers -> target chain (all at once)
+				if data.target_state.best_finalized_peer_at_best_self.0 < data.source_state.best_finalized_self.0 {
+					data.target_state.best_finalized_peer_at_best_self = data.source_state.best_finalized_self;
+				}
+				// syncing source headers -> target chain (all at once)
+				if data.source_state.best_finalized_peer_at_best_self.0 < data.target_state.best_finalized_self.0 {
+					data.source_state.best_finalized_peer_at_best_self = data.target_state.best_finalized_self;
+				}
+				// if target has received messages batch => increase blocks so that confirmations may be sent
+				if data.target_latest_received_nonce == 4
+					|| data.target_latest_received_nonce == 8
+					|| data.target_latest_received_nonce == 10
+				{
+					data.target_state.best_self =
+						HeaderId(data.target_state.best_self.0 + 1, data.target_state.best_self.0 + 1);
+					data.target_state.best_finalized_self = data.target_state.best_self;
+					data.source_state.best_self =
+						HeaderId(data.source_state.best_self.0 + 1, data.source_state.best_self.0 + 1);
+					data.source_state.best_finalized_self = data.source_state.best_self;
+				}
+				// if source has received all messages receiving confirmations => increase source block so that confirmations may be sent
+				if data.source_latest_confirmed_received_nonce == 10 {
+					exit_sender.unbounded_send(()).unwrap();
+				}
+			}),
+			exit_receiver.into_future().map(|(_, _)| ()),
+		);
+
+		// there are no strict restrictions on when reward confirmation should come
+		// (because `max_unconfirmed_nonces_at_target` is `100` in tests and this confirmation
+		// depends on the state of both clients)
+		// => we do not check it here
+		assert_eq!(result.submitted_messages_proofs[0].0, 1..=4);
+		assert_eq!(result.submitted_messages_proofs[1].0, 5..=8);
+		assert_eq!(result.submitted_messages_proofs[2].0, 9..=10);
+		assert!(!result.submitted_messages_receiving_proofs.is_empty());
+	}
+}

bridges/relays/generic/messages/src/message_race_delivery.rs

@@ -0,0 +1,871 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+//! Message delivery race delivers proof-of-messages from lane.source to lane.target.
+
+use crate::message_lane::{MessageLane, SourceHeaderIdOf, TargetHeaderIdOf};
+use crate::message_lane_loop::{
+	MessageDeliveryParams, MessageProofParameters, MessageWeightsMap, SourceClient as MessageLaneSourceClient,
+	SourceClientState, TargetClient as MessageLaneTargetClient, TargetClientState,
+};
+use crate::message_race_loop::{
+	MessageRace, NoncesRange, RaceState, RaceStrategy, SourceClient, SourceClientNonces, TargetClient,
+	TargetClientNonces,
+};
+use crate::message_race_strategy::BasicStrategy;
+use crate::metrics::MessageLaneLoopMetrics;
+
+use async_trait::async_trait;
+use bp_message_lane::{MessageNonce, UnrewardedRelayersState, Weight};
+use futures::stream::FusedStream;
+use relay_utils::FailedClient;
+use std::{
+	collections::{BTreeMap, VecDeque},
+	marker::PhantomData,
+	ops::RangeInclusive,
+	time::Duration,
+};
+
+/// Run message delivery race.
+pub async fn run<P: MessageLane>(
+	source_client: impl MessageLaneSourceClient<P>,
+	source_state_updates: impl FusedStream<Item = SourceClientState<P>>,
+	target_client: impl MessageLaneTargetClient<P>,
+	target_state_updates: impl FusedStream<Item = TargetClientState<P>>,
+	stall_timeout: Duration,
+	metrics_msg: Option<MessageLaneLoopMetrics>,
+	params: MessageDeliveryParams,
+) -> Result<(), FailedClient> {
+	crate::message_race_loop::run(
+		MessageDeliveryRaceSource {
+			client: source_client,
+			metrics_msg: metrics_msg.clone(),
+			_phantom: Default::default(),
+		},
+		source_state_updates,
+		MessageDeliveryRaceTarget {
+			client: target_client,
+			metrics_msg,
+			_phantom: Default::default(),
+		},
+		target_state_updates,
+		stall_timeout,
+		MessageDeliveryStrategy::<P> {
+			max_unrewarded_relayer_entries_at_target: params.max_unrewarded_relayer_entries_at_target,
+			max_unconfirmed_nonces_at_target: params.max_unconfirmed_nonces_at_target,
+			max_messages_in_single_batch: params.max_messages_in_single_batch,
+			max_messages_weight_in_single_batch: params.max_messages_weight_in_single_batch,
+			max_messages_size_in_single_batch: params.max_messages_size_in_single_batch,
+			latest_confirmed_nonces_at_source: VecDeque::new(),
+			target_nonces: None,
+			strategy: BasicStrategy::new(),
+		},
+	)
+	.await
+}
+
+/// Message delivery race.
+struct MessageDeliveryRace<P>(std::marker::PhantomData<P>);
+
+impl<P: MessageLane> MessageRace for MessageDeliveryRace<P> {
+	type SourceHeaderId = SourceHeaderIdOf<P>;
+	type TargetHeaderId = TargetHeaderIdOf<P>;
+
+	type MessageNonce = MessageNonce;
+	type Proof = P::MessagesProof;
+
+	fn source_name() -> String {
+		format!("{}::MessagesDelivery", P::SOURCE_NAME)
+	}
+
+	fn target_name() -> String {
+		format!("{}::MessagesDelivery", P::TARGET_NAME)
+	}
+}
+
+/// Message delivery race source, which is a source of the lane.
+struct MessageDeliveryRaceSource<P: MessageLane, C> {
+	client: C,
+	metrics_msg: Option<MessageLaneLoopMetrics>,
+	_phantom: PhantomData<P>,
+}
+
+#[async_trait]
+impl<P, C> SourceClient<MessageDeliveryRace<P>> for MessageDeliveryRaceSource<P, C>
+where
+	P: MessageLane,
+	C: MessageLaneSourceClient<P>,
+{
+	type Error = C::Error;
+	type NoncesRange = MessageWeightsMap;
+	type ProofParameters = MessageProofParameters;
+
+	async fn nonces(
+		&self,
+		at_block: SourceHeaderIdOf<P>,
+		prev_latest_nonce: MessageNonce,
+	) -> Result<(SourceHeaderIdOf<P>, SourceClientNonces<Self::NoncesRange>), Self::Error> {
+		let (at_block, latest_generated_nonce) = self.client.latest_generated_nonce(at_block).await?;
+		let (at_block, latest_confirmed_nonce) = self.client.latest_confirmed_received_nonce(at_block).await?;
+
+		if let Some(metrics_msg) = self.metrics_msg.as_ref() {
+			metrics_msg.update_source_latest_generated_nonce::<P>(latest_generated_nonce);
+			metrics_msg.update_source_latest_confirmed_nonce::<P>(latest_confirmed_nonce);
+		}
+
+		let new_nonces = if latest_generated_nonce > prev_latest_nonce {
+			self.client
+				.generated_messages_weights(at_block.clone(), prev_latest_nonce + 1..=latest_generated_nonce)
+				.await?
+		} else {
+			MessageWeightsMap::new()
+		};
+
+		Ok((
+			at_block,
+			SourceClientNonces {
+				new_nonces,
+				confirmed_nonce: Some(latest_confirmed_nonce),
+			},
+		))
+	}
+
+	async fn generate_proof(
+		&self,
+		at_block: SourceHeaderIdOf<P>,
+		nonces: RangeInclusive<MessageNonce>,
+		proof_parameters: Self::ProofParameters,
+	) -> Result<(SourceHeaderIdOf<P>, RangeInclusive<MessageNonce>, P::MessagesProof), Self::Error> {
+		self.client.prove_messages(at_block, nonces, proof_parameters).await
+	}
+}
+
+/// Message delivery race target, which is a target of the lane.
+struct MessageDeliveryRaceTarget<P: MessageLane, C> {
+	client: C,
+	metrics_msg: Option<MessageLaneLoopMetrics>,
+	_phantom: PhantomData<P>,
+}
+
+#[async_trait]
+impl<P, C> TargetClient<MessageDeliveryRace<P>> for MessageDeliveryRaceTarget<P, C>
+where
+	P: MessageLane,
+	C: MessageLaneTargetClient<P>,
+{
+	type Error = C::Error;
+	type TargetNoncesData = DeliveryRaceTargetNoncesData;
+
+	async fn nonces(
+		&self,
+		at_block: TargetHeaderIdOf<P>,
+		update_metrics: bool,
+	) -> Result<(TargetHeaderIdOf<P>, TargetClientNonces<DeliveryRaceTargetNoncesData>), Self::Error> {
+		let (at_block, latest_received_nonce) = self.client.latest_received_nonce(at_block).await?;
+		let (at_block, latest_confirmed_nonce) = self.client.latest_confirmed_received_nonce(at_block).await?;
+		let (at_block, unrewarded_relayers) = self.client.unrewarded_relayers_state(at_block).await?;
+
+		if update_metrics {
+			if let Some(metrics_msg) = self.metrics_msg.as_ref() {
+				metrics_msg.update_target_latest_received_nonce::<P>(latest_received_nonce);
+				metrics_msg.update_target_latest_confirmed_nonce::<P>(latest_confirmed_nonce);
+			}
+		}
+
+		Ok((
+			at_block,
+			TargetClientNonces {
+				latest_nonce: latest_received_nonce,
+				nonces_data: DeliveryRaceTargetNoncesData {
+					confirmed_nonce: latest_confirmed_nonce,
+					unrewarded_relayers,
+				},
+			},
+		))
+	}
+
+	async fn submit_proof(
+		&self,
+		generated_at_block: SourceHeaderIdOf<P>,
+		nonces: RangeInclusive<MessageNonce>,
+		proof: P::MessagesProof,
+	) -> Result<RangeInclusive<MessageNonce>, Self::Error> {
+		self.client
+			.submit_messages_proof(generated_at_block, nonces, proof)
+			.await
+	}
+}
+
+/// Additional nonces data from the target client used by message delivery race.
+#[derive(Debug, Clone)]
+struct DeliveryRaceTargetNoncesData {
+	/// Latest nonce that we know: (1) has been delivered to us (2) has been confirmed
+	/// back to the source node (by confirmations race) and (3) relayer has received
+	/// reward for (and this has been confirmed by the message delivery race).
+	confirmed_nonce: MessageNonce,
+	/// State of the unrewarded relayers set at the target node.
+	unrewarded_relayers: UnrewardedRelayersState,
+}
+
+/// Messages delivery strategy.
+struct MessageDeliveryStrategy<P: MessageLane> {
+	/// Maximal unrewarded relayer entries at target client.
+	max_unrewarded_relayer_entries_at_target: MessageNonce,
+	/// Maximal unconfirmed nonces at target client.
+	max_unconfirmed_nonces_at_target: MessageNonce,
+	/// Maximal number of messages in the single delivery transaction.
+	max_messages_in_single_batch: MessageNonce,
+	/// Maximal cumulative messages weight in the single delivery transaction.
+	max_messages_weight_in_single_batch: Weight,
+	/// Maximal messages size in the single delivery transaction.
+	max_messages_size_in_single_batch: usize,
+	/// Latest confirmed nonces at the source client + the header id where we have first met this nonce.
+	latest_confirmed_nonces_at_source: VecDeque<(SourceHeaderIdOf<P>, MessageNonce)>,
+	/// Target nonces from the source client.
+	target_nonces: Option<TargetClientNonces<DeliveryRaceTargetNoncesData>>,
+	/// Basic delivery strategy.
+	strategy: MessageDeliveryStrategyBase<P>,
+}
+
+type MessageDeliveryStrategyBase<P> = BasicStrategy<
+	<P as MessageLane>::SourceHeaderNumber,
+	<P as MessageLane>::SourceHeaderHash,
+	<P as MessageLane>::TargetHeaderNumber,
+	<P as MessageLane>::TargetHeaderHash,
+	MessageWeightsMap,
+	<P as MessageLane>::MessagesProof,
+>;
+
+impl<P: MessageLane> std::fmt::Debug for MessageDeliveryStrategy<P> {
+	fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+		fmt.debug_struct("MessageDeliveryStrategy")
+			.field(
+				"max_unrewarded_relayer_entries_at_target",
+				&self.max_unrewarded_relayer_entries_at_target,
+			)
+			.field(
+				"max_unconfirmed_nonces_at_target",
+				&self.max_unconfirmed_nonces_at_target,
+			)
+			.field("max_messages_in_single_batch", &self.max_messages_in_single_batch)
+			.field(
+				"max_messages_weight_in_single_batch",
+				&self.max_messages_weight_in_single_batch,
+			)
+			.field(
+				"max_messages_size_in_single_batch",
+				&self.max_messages_size_in_single_batch,
+			)
+			.field(
+				"latest_confirmed_nonces_at_source",
+				&self.latest_confirmed_nonces_at_source,
+			)
+			.field("target_nonces", &self.target_nonces)
+			.field("strategy", &self.strategy)
+			.finish()
+	}
+}
+
+impl<P: MessageLane> RaceStrategy<SourceHeaderIdOf<P>, TargetHeaderIdOf<P>, P::MessagesProof>
+	for MessageDeliveryStrategy<P>
+{
+	type SourceNoncesRange = MessageWeightsMap;
+	type ProofParameters = MessageProofParameters;
+	type TargetNoncesData = DeliveryRaceTargetNoncesData;
+
+	fn is_empty(&self) -> bool {
+		self.strategy.is_empty()
+	}
+
+	fn best_at_source(&self) -> Option<MessageNonce> {
+		self.strategy.best_at_source()
+	}
+
+	fn best_at_target(&self) -> Option<MessageNonce> {
+		self.strategy.best_at_target()
+	}
+
+	fn source_nonces_updated(
+		&mut self,
+		at_block: SourceHeaderIdOf<P>,
+		nonces: SourceClientNonces<Self::SourceNoncesRange>,
+	) {
+		if let Some(confirmed_nonce) = nonces.confirmed_nonce {
+			let is_confirmed_nonce_updated = self
+				.latest_confirmed_nonces_at_source
+				.back()
+				.map(|(_, prev_nonce)| *prev_nonce != confirmed_nonce)
+				.unwrap_or(true);
+			if is_confirmed_nonce_updated {
+				self.latest_confirmed_nonces_at_source
+					.push_back((at_block.clone(), confirmed_nonce));
+			}
+		}
+		self.strategy.source_nonces_updated(at_block, nonces)
+	}
+
+	fn best_target_nonces_updated(
+		&mut self,
+		nonces: TargetClientNonces<DeliveryRaceTargetNoncesData>,
+		race_state: &mut RaceState<SourceHeaderIdOf<P>, TargetHeaderIdOf<P>, P::MessagesProof>,
+	) {
+		// best target nonces must always be ge than finalized target nonces
+		let mut target_nonces = self.target_nonces.take().unwrap_or_else(|| nonces.clone());
+		target_nonces.nonces_data = nonces.nonces_data.clone();
+		target_nonces.latest_nonce = std::cmp::max(target_nonces.latest_nonce, nonces.latest_nonce);
+		self.target_nonces = Some(target_nonces);
+
+		self.strategy.best_target_nonces_updated(
+			TargetClientNonces {
+				latest_nonce: nonces.latest_nonce,
+				nonces_data: (),
+			},
+			race_state,
+		)
+	}
+
+	fn finalized_target_nonces_updated(
+		&mut self,
+		nonces: TargetClientNonces<DeliveryRaceTargetNoncesData>,
+		race_state: &mut RaceState<SourceHeaderIdOf<P>, TargetHeaderIdOf<P>, P::MessagesProof>,
+	) {
+		if let Some(ref best_finalized_source_header_id_at_best_target) =
+			race_state.best_finalized_source_header_id_at_best_target
+		{
+			let oldest_header_number_to_keep = best_finalized_source_header_id_at_best_target.0;
+			while self
+				.latest_confirmed_nonces_at_source
+				.front()
+				.map(|(id, _)| id.0 < oldest_header_number_to_keep)
+				.unwrap_or(false)
+			{
+				self.latest_confirmed_nonces_at_source.pop_front();
+			}
+		}
+
+		if let Some(ref mut target_nonces) = self.target_nonces {
+			target_nonces.latest_nonce = std::cmp::max(target_nonces.latest_nonce, nonces.latest_nonce);
+		}
+
+		self.strategy.finalized_target_nonces_updated(
+			TargetClientNonces {
+				latest_nonce: nonces.latest_nonce,
+				nonces_data: (),
+			},
+			race_state,
+		)
+	}
+
+	fn select_nonces_to_deliver(
+		&mut self,
+		race_state: &RaceState<SourceHeaderIdOf<P>, TargetHeaderIdOf<P>, P::MessagesProof>,
+	) -> Option<(RangeInclusive<MessageNonce>, Self::ProofParameters)> {
+		let best_finalized_source_header_id_at_best_target =
+			race_state.best_finalized_source_header_id_at_best_target.clone()?;
+		let latest_confirmed_nonce_at_source = self
+			.latest_confirmed_nonces_at_source
+			.iter()
+			.take_while(|(id, _)| id.0 <= best_finalized_source_header_id_at_best_target.0)
+			.last()
+			.map(|(_, nonce)| *nonce)?;
+		let target_nonces = self.target_nonces.as_ref()?;
+
+		// There's additional condition in the message delivery race: target would reject messages
+		// if there are too much unconfirmed messages at the inbound lane.
+
+		// The receiving race is responsible to deliver confirmations back to the source chain. So if
+		// there's a lot of unconfirmed messages, let's wait until it'll be able to do its job.
+		let latest_received_nonce_at_target = target_nonces.latest_nonce;
+		let confirmations_missing = latest_received_nonce_at_target.checked_sub(latest_confirmed_nonce_at_source);
+		match confirmations_missing {
+			Some(confirmations_missing) if confirmations_missing >= self.max_unconfirmed_nonces_at_target => {
+				log::debug!(
+					target: "bridge",
+					"Cannot deliver any more messages from {} to {}. Too many unconfirmed nonces \
+					at target: target.latest_received={:?}, source.latest_confirmed={:?}, max={:?}",
+					MessageDeliveryRace::<P>::source_name(),
+					MessageDeliveryRace::<P>::target_name(),
+					latest_received_nonce_at_target,
+					latest_confirmed_nonce_at_source,
+					self.max_unconfirmed_nonces_at_target,
+				);
+
+				return None;
+			}
+			_ => (),
+		}
+
+		// Ok - we may have new nonces to deliver. But target may still reject new messages, because we haven't
+		// notified it that (some) messages have been confirmed. So we may want to include updated
+		// `source.latest_confirmed` in the proof.
+		//
+		// Important note: we're including outbound state lane proof whenever there are unconfirmed nonces
+		// on the target chain. Other strategy is to include it only if it's absolutely necessary.
+		let latest_confirmed_nonce_at_target = target_nonces.nonces_data.confirmed_nonce;
+		let outbound_state_proof_required = latest_confirmed_nonce_at_target < latest_confirmed_nonce_at_source;
+
+		// The target node would also reject messages if there are too many entries in the
+		// "unrewarded relayers" set. If we are unable to prove new rewards to the target node, then
+		// we should wait for confirmations race.
+		let unrewarded_relayer_entries_limit_reached =
+			target_nonces.nonces_data.unrewarded_relayers.unrewarded_relayer_entries
+				>= self.max_unrewarded_relayer_entries_at_target;
+		if unrewarded_relayer_entries_limit_reached {
+			// so there are already too many unrewarded relayer entries in the set
+			//
+			// => check if we can prove enough rewards. If not, we should wait for more rewards to be paid
+			let number_of_rewards_being_proved =
+				latest_confirmed_nonce_at_source.saturating_sub(latest_confirmed_nonce_at_target);
+			let enough_rewards_being_proved = number_of_rewards_being_proved
+				>= target_nonces.nonces_data.unrewarded_relayers.messages_in_oldest_entry;
+			if !enough_rewards_being_proved {
+				return None;
+			}
+		}
+
+		// If we're here, then the confirmations race did its job && sending side now knows that messages
+		// have been delivered. Now let's select nonces that we want to deliver.
+		//
+		// We may deliver at most:
+		//
+		// max_unconfirmed_nonces_at_target - (latest_received_nonce_at_target - latest_confirmed_nonce_at_target)
+		//
+		// messages in the batch. But since we're including outbound state proof in the batch, then it
+		// may be increased to:
+		//
+		// max_unconfirmed_nonces_at_target - (latest_received_nonce_at_target - latest_confirmed_nonce_at_source)
+		let future_confirmed_nonce_at_target = if outbound_state_proof_required {
+			latest_confirmed_nonce_at_source
+		} else {
+			latest_confirmed_nonce_at_target
+		};
+		let max_nonces = latest_received_nonce_at_target
+			.checked_sub(future_confirmed_nonce_at_target)
+			.and_then(|diff| self.max_unconfirmed_nonces_at_target.checked_sub(diff))
+			.unwrap_or_default();
+		let max_nonces = std::cmp::min(max_nonces, self.max_messages_in_single_batch);
+		let max_messages_weight_in_single_batch = self.max_messages_weight_in_single_batch;
+		let max_messages_size_in_single_batch = self.max_messages_size_in_single_batch;
+		let mut selected_weight: Weight = 0;
+		let mut selected_size: usize = 0;
+		let mut selected_count: MessageNonce = 0;
+
+		let selected_nonces = self
+			.strategy
+			.select_nonces_to_deliver_with_selector(race_state, |range| {
+				let to_requeue = range
+					.into_iter()
+					.skip_while(|(_, weight)| {
+						// Since we (hopefully) have some reserves in `max_messages_weight_in_single_batch`
+						// and `max_messages_size_in_single_batch`, we may still try to submit transaction
+						// with single message if message overflows these limits. The worst case would be if
+						// transaction will be rejected by the target runtime, but at least we have tried.
+
+						// limit messages in the batch by weight
+						let new_selected_weight = match selected_weight.checked_add(weight.weight) {
+							Some(new_selected_weight) if new_selected_weight <= max_messages_weight_in_single_batch => {
+								new_selected_weight
+							}
+							new_selected_weight if selected_count == 0 => {
+								log::warn!(
+									target: "bridge",
+									"Going to submit message delivery transaction with declared dispatch \
+									weight {:?} that overflows maximal configured weight {}",
+									new_selected_weight,
+									max_messages_weight_in_single_batch,
+								);
+								new_selected_weight.unwrap_or(Weight::MAX)
+							}
+							_ => return false,
+						};
+
+						// limit messages in the batch by size
+						let new_selected_size = match selected_size.checked_add(weight.size) {
+							Some(new_selected_size) if new_selected_size <= max_messages_size_in_single_batch => {
+								new_selected_size
+							}
+							new_selected_size if selected_count == 0 => {
+								log::warn!(
+									target: "bridge",
+									"Going to submit message delivery transaction with message \
+									size {:?} that overflows maximal configured size {}",
+									new_selected_size,
+									max_messages_size_in_single_batch,
+								);
+								new_selected_size.unwrap_or(usize::MAX)
+							}
+							_ => return false,
+						};
+
+						// limit number of messages in the batch
+						let new_selected_count = selected_count + 1;
+						if new_selected_count > max_nonces {
+							return false;
+						}
+
+						selected_weight = new_selected_weight;
+						selected_size = new_selected_size;
+						selected_count = new_selected_count;
+						true
+					})
+					.collect::<BTreeMap<_, _>>();
+				if to_requeue.is_empty() {
+					None
+				} else {
+					Some(to_requeue)
+				}
+			})?;
+
+		Some((
+			selected_nonces,
+			MessageProofParameters {
+				outbound_state_proof_required,
+				dispatch_weight: selected_weight,
+			},
+		))
+	}
+}
+
+impl NoncesRange for MessageWeightsMap {
+	fn begin(&self) -> MessageNonce {
+		self.keys().next().cloned().unwrap_or_default()
+	}
+
+	fn end(&self) -> MessageNonce {
+		self.keys().next_back().cloned().unwrap_or_default()
+	}
+
+	fn greater_than(mut self, nonce: MessageNonce) -> Option<Self> {
+		let gte = self.split_off(&(nonce + 1));
+		if gte.is_empty() {
+			None
+		} else {
+			Some(gte)
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::message_lane_loop::{
+		tests::{header_id, TestMessageLane, TestMessagesProof, TestSourceHeaderId, TestTargetHeaderId},
+		MessageWeights,
+	};
+
+	type TestRaceState = RaceState<TestSourceHeaderId, TestTargetHeaderId, TestMessagesProof>;
+	type TestStrategy = MessageDeliveryStrategy<TestMessageLane>;
+
+	fn prepare_strategy() -> (TestRaceState, TestStrategy) {
+		let mut race_state = RaceState {
+			best_finalized_source_header_id_at_source: Some(header_id(1)),
+			best_finalized_source_header_id_at_best_target: Some(header_id(1)),
+			best_target_header_id: Some(header_id(1)),
+			best_finalized_target_header_id: Some(header_id(1)),
+			nonces_to_submit: None,
+			nonces_submitted: None,
+		};
+
+		let mut race_strategy = TestStrategy {
+			max_unrewarded_relayer_entries_at_target: 4,
+			max_unconfirmed_nonces_at_target: 4,
+			max_messages_in_single_batch: 4,
+			max_messages_weight_in_single_batch: 4,
+			max_messages_size_in_single_batch: 4,
+			latest_confirmed_nonces_at_source: vec![(header_id(1), 19)].into_iter().collect(),
+			target_nonces: Some(TargetClientNonces {
+				latest_nonce: 19,
+				nonces_data: DeliveryRaceTargetNoncesData {
+					confirmed_nonce: 19,
+					unrewarded_relayers: UnrewardedRelayersState {
+						unrewarded_relayer_entries: 0,
+						messages_in_oldest_entry: 0,
+						total_messages: 0,
+					},
+				},
+			}),
+			strategy: BasicStrategy::new(),
+		};
+
+		race_strategy.strategy.source_nonces_updated(
+			header_id(1),
+			SourceClientNonces {
+				new_nonces: vec![
+					(20, MessageWeights { weight: 1, size: 1 }),
+					(21, MessageWeights { weight: 1, size: 1 }),
+					(22, MessageWeights { weight: 1, size: 1 }),
+					(23, MessageWeights { weight: 1, size: 1 }),
+				]
+				.into_iter()
+				.collect(),
+				confirmed_nonce: Some(19),
+			},
+		);
+
+		let target_nonces = TargetClientNonces {
+			latest_nonce: 19,
+			nonces_data: (),
+		};
+		race_strategy
+			.strategy
+			.best_target_nonces_updated(target_nonces.clone(), &mut race_state);
+		race_strategy
+			.strategy
+			.finalized_target_nonces_updated(target_nonces, &mut race_state);
+
+		(race_state, race_strategy)
+	}
+
+	fn proof_parameters(state_required: bool, weight: Weight) -> MessageProofParameters {
+		MessageProofParameters {
+			outbound_state_proof_required: state_required,
+			dispatch_weight: weight,
+		}
+	}
+
+	#[test]
+	fn weights_map_works_as_nonces_range() {
+		fn build_map(range: RangeInclusive<MessageNonce>) -> MessageWeightsMap {
+			range
+				.map(|idx| {
+					(
+						idx,
+						MessageWeights {
+							weight: idx,
+							size: idx as _,
+						},
+					)
+				})
+				.collect()
+		}
+
+		let map = build_map(20..=30);
+
+		assert_eq!(map.begin(), 20);
+		assert_eq!(map.end(), 30);
+		assert_eq!(map.clone().greater_than(10), Some(build_map(20..=30)));
+		assert_eq!(map.clone().greater_than(19), Some(build_map(20..=30)));
+		assert_eq!(map.clone().greater_than(20), Some(build_map(21..=30)));
+		assert_eq!(map.clone().greater_than(25), Some(build_map(26..=30)));
+		assert_eq!(map.clone().greater_than(29), Some(build_map(30..=30)));
+		assert_eq!(map.greater_than(30), None);
+	}
+
+	#[test]
+	fn message_delivery_strategy_selects_messages_to_deliver() {
+		let (state, mut strategy) = prepare_strategy();
+
+		// both sides are ready to relay new messages
+		assert_eq!(
+			strategy.select_nonces_to_deliver(&state),
+			Some(((20..=23), proof_parameters(false, 4)))
+		);
+	}
+
+	#[test]
+	fn message_delivery_strategy_selects_nothing_if_too_many_confirmations_missing() {
+		let (state, mut strategy) = prepare_strategy();
+
+		// if there are already `max_unconfirmed_nonces_at_target` messages on target,
+		// we need to wait until confirmations will be delivered by receiving race
+		strategy.latest_confirmed_nonces_at_source = vec![(
+			header_id(1),
+			strategy.target_nonces.as_ref().unwrap().latest_nonce - strategy.max_unconfirmed_nonces_at_target,
+		)]
+		.into_iter()
+		.collect();
+		assert_eq!(strategy.select_nonces_to_deliver(&state), None);
+	}
+
+	#[test]
+	fn message_delivery_strategy_includes_outbound_state_proof_when_new_nonces_are_available() {
+		let (state, mut strategy) = prepare_strategy();
+
+		// if there are new confirmed nonces on source, we want to relay this information
+		// to target to prune rewards queue
+		let prev_confirmed_nonce_at_source = strategy.latest_confirmed_nonces_at_source.back().unwrap().1;
+		strategy.target_nonces.as_mut().unwrap().nonces_data.confirmed_nonce = prev_confirmed_nonce_at_source - 1;
+		assert_eq!(
+			strategy.select_nonces_to_deliver(&state),
+			Some(((20..=23), proof_parameters(true, 4)))
+		);
+	}
+
+	#[test]
+	fn message_delivery_strategy_selects_nothing_if_there_are_too_many_unrewarded_relayers() {
+		let (state, mut strategy) = prepare_strategy();
+
+		// if there are already `max_unrewarded_relayer_entries_at_target` entries at target,
+		// we need to wait until rewards will be paid
+		{
+			let mut unrewarded_relayers = &mut strategy.target_nonces.as_mut().unwrap().nonces_data.unrewarded_relayers;
+			unrewarded_relayers.unrewarded_relayer_entries = strategy.max_unrewarded_relayer_entries_at_target;
+			unrewarded_relayers.messages_in_oldest_entry = 4;
+		}
+		assert_eq!(strategy.select_nonces_to_deliver(&state), None);
+	}
+
+	#[test]
+	fn message_delivery_strategy_selects_nothing_if_proved_rewards_is_not_enough_to_remove_oldest_unrewarded_entry() {
+		let (state, mut strategy) = prepare_strategy();
+
+		// if there are already `max_unrewarded_relayer_entries_at_target` entries at target,
+		// we need to prove at least `messages_in_oldest_entry` rewards
+		let prev_confirmed_nonce_at_source = strategy.latest_confirmed_nonces_at_source.back().unwrap().1;
+		{
+			let mut nonces_data = &mut strategy.target_nonces.as_mut().unwrap().nonces_data;
+			nonces_data.confirmed_nonce = prev_confirmed_nonce_at_source - 1;
+			let mut unrewarded_relayers = &mut nonces_data.unrewarded_relayers;
+			unrewarded_relayers.unrewarded_relayer_entries = strategy.max_unrewarded_relayer_entries_at_target;
+			unrewarded_relayers.messages_in_oldest_entry = 4;
+		}
+		assert_eq!(strategy.select_nonces_to_deliver(&state), None);
+	}
+
+	#[test]
+	fn message_delivery_strategy_includes_outbound_state_proof_if_proved_rewards_is_enough() {
+		let (state, mut strategy) = prepare_strategy();
+
+		// if there are already `max_unrewarded_relayer_entries_at_target` entries at target,
+		// we need to prove at least `messages_in_oldest_entry` rewards
+		let prev_confirmed_nonce_at_source = strategy.latest_confirmed_nonces_at_source.back().unwrap().1;
+		{
+			let mut nonces_data = &mut strategy.target_nonces.as_mut().unwrap().nonces_data;
+			nonces_data.confirmed_nonce = prev_confirmed_nonce_at_source - 3;
+			let mut unrewarded_relayers = &mut nonces_data.unrewarded_relayers;
+			unrewarded_relayers.unrewarded_relayer_entries = strategy.max_unrewarded_relayer_entries_at_target;
+			unrewarded_relayers.messages_in_oldest_entry = 3;
+		}
+		assert_eq!(
+			strategy.select_nonces_to_deliver(&state),
+			Some(((20..=23), proof_parameters(true, 4)))
+		);
+	}
+
+	#[test]
+	fn message_delivery_strategy_limits_batch_by_messages_weight() {
+		let (state, mut strategy) = prepare_strategy();
+
+		// not all queued messages may fit in the batch, because batch has max weight
+		strategy.max_messages_weight_in_single_batch = 3;
+		assert_eq!(
+			strategy.select_nonces_to_deliver(&state),
+			Some(((20..=22), proof_parameters(false, 3)))
+		);
+	}
+
+	#[test]
+	fn message_delivery_strategy_accepts_single_message_even_if_its_weight_overflows_maximal_weight() {
+		let (state, mut strategy) = prepare_strategy();
+
+		// first message doesn't fit in the batch, because it has weight (10) that overflows max weight (4)
+		strategy.strategy.source_queue_mut()[0].1.get_mut(&20).unwrap().weight = 10;
+		assert_eq!(
+			strategy.select_nonces_to_deliver(&state),
+			Some(((20..=20), proof_parameters(false, 10)))
+		);
+	}
+
+	#[test]
+	fn message_delivery_strategy_limits_batch_by_messages_size() {
+		let (state, mut strategy) = prepare_strategy();
+
+		// not all queued messages may fit in the batch, because batch has max weight
+		strategy.max_messages_size_in_single_batch = 3;
+		assert_eq!(
+			strategy.select_nonces_to_deliver(&state),
+			Some(((20..=22), proof_parameters(false, 3)))
+		);
+	}
+
+	#[test]
+	fn message_delivery_strategy_accepts_single_message_even_if_its_weight_overflows_maximal_size() {
+		let (state, mut strategy) = prepare_strategy();
+
+		// first message doesn't fit in the batch, because it has weight (10) that overflows max weight (4)
+		strategy.strategy.source_queue_mut()[0].1.get_mut(&20).unwrap().size = 10;
+		assert_eq!(
+			strategy.select_nonces_to_deliver(&state),
+			Some(((20..=20), proof_parameters(false, 1)))
+		);
+	}
+
+	#[test]
+	fn message_delivery_strategy_limits_batch_by_messages_count_when_there_is_upper_limit() {
+		let (state, mut strategy) = prepare_strategy();
+
+		// not all queued messages may fit in the batch, because batch has max number of messages limit
+		strategy.max_messages_in_single_batch = 3;
+		assert_eq!(
+			strategy.select_nonces_to_deliver(&state),
+			Some(((20..=22), proof_parameters(false, 3)))
+		);
+	}
+
+	#[test]
+	fn message_delivery_strategy_limits_batch_by_messages_count_when_there_are_unconfirmed_nonces() {
+		let (state, mut strategy) = prepare_strategy();
+
+		// 1 delivery confirmation from target to source is still missing, so we may only
+		// relay 3 new messages
+		let prev_confirmed_nonce_at_source = strategy.latest_confirmed_nonces_at_source.back().unwrap().1;
+		strategy.latest_confirmed_nonces_at_source = vec![(header_id(1), prev_confirmed_nonce_at_source - 1)]
+			.into_iter()
+			.collect();
+		strategy.target_nonces.as_mut().unwrap().nonces_data.confirmed_nonce = prev_confirmed_nonce_at_source - 1;
+		assert_eq!(
+			strategy.select_nonces_to_deliver(&state),
+			Some(((20..=22), proof_parameters(false, 3)))
+		);
+	}
+
+	#[test]
+	fn message_delivery_strategy_waits_for_confirmed_nonce_header_to_appear_on_target() {
+		// 1 delivery confirmation from target to source is still missing, so we may deliver
+		// reward confirmation with our message delivery transaction. But the problem is that
+		// the reward has been paid at header 2 && this header is still unknown to target node.
+		//
+		// => so we can't deliver more than 3 messages
+		let (mut state, mut strategy) = prepare_strategy();
+		let prev_confirmed_nonce_at_source = strategy.latest_confirmed_nonces_at_source.back().unwrap().1;
+		strategy.latest_confirmed_nonces_at_source = vec![
+			(header_id(1), prev_confirmed_nonce_at_source - 1),
+			(header_id(2), prev_confirmed_nonce_at_source),
+		]
+		.into_iter()
+		.collect();
+		strategy.target_nonces.as_mut().unwrap().nonces_data.confirmed_nonce = prev_confirmed_nonce_at_source - 1;
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(1));
+		assert_eq!(
+			strategy.select_nonces_to_deliver(&state),
+			Some(((20..=22), proof_parameters(false, 3)))
+		);
+
+		// the same situation, but the header 2 is known to the target node, so we may deliver reward confirmation
+		let (mut state, mut strategy) = prepare_strategy();
+		let prev_confirmed_nonce_at_source = strategy.latest_confirmed_nonces_at_source.back().unwrap().1;
+		strategy.latest_confirmed_nonces_at_source = vec![
+			(header_id(1), prev_confirmed_nonce_at_source - 1),
+			(header_id(2), prev_confirmed_nonce_at_source),
+		]
+		.into_iter()
+		.collect();
+		strategy.target_nonces.as_mut().unwrap().nonces_data.confirmed_nonce = prev_confirmed_nonce_at_source - 1;
+		state.best_finalized_source_header_id_at_source = Some(header_id(2));
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(2));
+		assert_eq!(
+			strategy.select_nonces_to_deliver(&state),
+			Some(((20..=23), proof_parameters(true, 4)))
+		);
+	}
+}

bridges/relays/generic/messages/src/message_race_loop.rs

@@ -0,0 +1,612 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+//! Loop that is serving single race within message lane. This could be
+//! message delivery race, receiving confirmations race or processing
+//! confirmations race.
+//!
+//! The idea of the race is simple - we have `nonce`-s on source and target
+//! nodes. We're trying to prove that the source node has this nonce (and
+//! associated data - like messages, lane state, etc) to the target node by
+//! generating and submitting proof.
+
+use crate::message_lane_loop::ClientState;
+
+use async_trait::async_trait;
+use bp_message_lane::MessageNonce;
+use futures::{
+	future::FutureExt,
+	stream::{FusedStream, StreamExt},
+};
+use relay_utils::{process_future_result, retry_backoff, FailedClient, MaybeConnectionError};
+use std::{
+	fmt::Debug,
+	ops::RangeInclusive,
+	time::{Duration, Instant},
+};
+
+/// One of races within lane.
+pub trait MessageRace {
+	/// Header id of the race source.
+	type SourceHeaderId: Debug + Clone + PartialEq;
+	/// Header id of the race source.
+	type TargetHeaderId: Debug + Clone + PartialEq;
+
+	/// Message nonce used in the race.
+	type MessageNonce: Debug + Clone;
+	/// Proof that is generated and delivered in this race.
+	type Proof: Debug + Clone;
+
+	/// Name of the race source.
+	fn source_name() -> String;
+	/// Name of the race target.
+	fn target_name() -> String;
+}
+
+/// State of race source client.
+type SourceClientState<P> = ClientState<<P as MessageRace>::SourceHeaderId, <P as MessageRace>::TargetHeaderId>;
+
+/// State of race target client.
+type TargetClientState<P> = ClientState<<P as MessageRace>::TargetHeaderId, <P as MessageRace>::SourceHeaderId>;
+
+/// Inclusive nonces range.
+pub trait NoncesRange: Debug + Sized {
+	/// Get begin of the range.
+	fn begin(&self) -> MessageNonce;
+	/// Get end of the range.
+	fn end(&self) -> MessageNonce;
+	/// Returns new range with current range nonces that are greater than the passed `nonce`.
+	/// If there are no such nonces, `None` is returned.
+	fn greater_than(self, nonce: MessageNonce) -> Option<Self>;
+}
+
+/// Nonces on the race source client.
+#[derive(Debug, Clone)]
+pub struct SourceClientNonces<NoncesRange> {
+	/// New nonces range known to the client. `New` here means all nonces generated after
+	/// `prev_latest_nonce` passed to the `SourceClient::nonces` method.
+	pub new_nonces: NoncesRange,
+	/// Latest nonce that is confirmed to the bridged client. This nonce only makes
+	/// sense in some races. In other races it is `None`.
+	pub confirmed_nonce: Option<MessageNonce>,
+}
+
+/// Nonces on the race target client.
+#[derive(Debug, Clone)]
+pub struct TargetClientNonces<TargetNoncesData> {
+	/// Latest nonce that is known to the target client.
+	pub latest_nonce: MessageNonce,
+	/// Additional data from target node that may be used by the race.
+	pub nonces_data: TargetNoncesData,
+}
+
+/// One of message lane clients, which is source client for the race.
+#[async_trait]
+pub trait SourceClient<P: MessageRace> {
+	/// Type of error this clients returns.
+	type Error: std::fmt::Debug + MaybeConnectionError;
+	/// Type of nonces range returned by the source client.
+	type NoncesRange: NoncesRange;
+	/// Additional proof parameters required to generate proof.
+	type ProofParameters;
+
+	/// Return nonces that are known to the source client.
+	async fn nonces(
+		&self,
+		at_block: P::SourceHeaderId,
+		prev_latest_nonce: MessageNonce,
+	) -> Result<(P::SourceHeaderId, SourceClientNonces<Self::NoncesRange>), Self::Error>;
+	/// Generate proof for delivering to the target client.
+	async fn generate_proof(
+		&self,
+		at_block: P::SourceHeaderId,
+		nonces: RangeInclusive<MessageNonce>,
+		proof_parameters: Self::ProofParameters,
+	) -> Result<(P::SourceHeaderId, RangeInclusive<MessageNonce>, P::Proof), Self::Error>;
+}
+
+/// One of message lane clients, which is target client for the race.
+#[async_trait]
+pub trait TargetClient<P: MessageRace> {
+	/// Type of error this clients returns.
+	type Error: std::fmt::Debug + MaybeConnectionError;
+	/// Type of the additional data from the target client, used by the race.
+	type TargetNoncesData: std::fmt::Debug;
+
+	/// Return nonces that are known to the target client.
+	async fn nonces(
+		&self,
+		at_block: P::TargetHeaderId,
+		update_metrics: bool,
+	) -> Result<(P::TargetHeaderId, TargetClientNonces<Self::TargetNoncesData>), Self::Error>;
+	/// Submit proof to the target client.
+	async fn submit_proof(
+		&self,
+		generated_at_block: P::SourceHeaderId,
+		nonces: RangeInclusive<MessageNonce>,
+		proof: P::Proof,
+	) -> Result<RangeInclusive<MessageNonce>, Self::Error>;
+}
+
+/// Race strategy.
+pub trait RaceStrategy<SourceHeaderId, TargetHeaderId, Proof>: Debug {
+	/// Type of nonces range expected from the source client.
+	type SourceNoncesRange: NoncesRange;
+	/// Additional proof parameters required to generate proof.
+	type ProofParameters;
+	/// Additional data expected from the target client.
+	type TargetNoncesData;
+
+	/// Should return true if nothing has to be synced.
+	fn is_empty(&self) -> bool;
+	/// Return best nonce at source node.
+	///
+	/// `Some` is returned only if we are sure that the value is greater or equal
+	/// than the result of `best_at_target`.
+	fn best_at_source(&self) -> Option<MessageNonce>;
+	/// Return best nonce at target node.
+	///
+	/// May return `None` if value is yet unknown.
+	fn best_at_target(&self) -> Option<MessageNonce>;
+
+	/// Called when nonces are updated at source node of the race.
+	fn source_nonces_updated(&mut self, at_block: SourceHeaderId, nonces: SourceClientNonces<Self::SourceNoncesRange>);
+	/// Called when best nonces are updated at target node of the race.
+	fn best_target_nonces_updated(
+		&mut self,
+		nonces: TargetClientNonces<Self::TargetNoncesData>,
+		race_state: &mut RaceState<SourceHeaderId, TargetHeaderId, Proof>,
+	);
+	/// Called when finalized nonces are updated at target node of the race.
+	fn finalized_target_nonces_updated(
+		&mut self,
+		nonces: TargetClientNonces<Self::TargetNoncesData>,
+		race_state: &mut RaceState<SourceHeaderId, TargetHeaderId, Proof>,
+	);
+	/// Should return `Some(nonces)` if we need to deliver proof of `nonces` (and associated
+	/// data) from source to target node.
+	/// Additionally, parameters required to generate proof are returned.
+	fn select_nonces_to_deliver(
+		&mut self,
+		race_state: &RaceState<SourceHeaderId, TargetHeaderId, Proof>,
+	) -> Option<(RangeInclusive<MessageNonce>, Self::ProofParameters)>;
+}
+
+/// State of the race.
+#[derive(Debug)]
+pub struct RaceState<SourceHeaderId, TargetHeaderId, Proof> {
+	/// Best finalized source header id at the source client.
+	pub best_finalized_source_header_id_at_source: Option<SourceHeaderId>,
+	/// Best finalized source header id at the best block on the target
+	/// client (at the `best_finalized_source_header_id_at_best_target`).
+	pub best_finalized_source_header_id_at_best_target: Option<SourceHeaderId>,
+	/// Best header id at the target client.
+	pub best_target_header_id: Option<TargetHeaderId>,
+	/// Best finalized header id at the target client.
+	pub best_finalized_target_header_id: Option<TargetHeaderId>,
+	/// Range of nonces that we have selected to submit.
+	pub nonces_to_submit: Option<(SourceHeaderId, RangeInclusive<MessageNonce>, Proof)>,
+	/// Range of nonces that is currently submitted.
+	pub nonces_submitted: Option<RangeInclusive<MessageNonce>>,
+}
+
+/// Run race loop until connection with target or source node is lost.
+pub async fn run<P: MessageRace, SC: SourceClient<P>, TC: TargetClient<P>>(
+	race_source: SC,
+	race_source_updated: impl FusedStream<Item = SourceClientState<P>>,
+	race_target: TC,
+	race_target_updated: impl FusedStream<Item = TargetClientState<P>>,
+	stall_timeout: Duration,
+	mut strategy: impl RaceStrategy<
+		P::SourceHeaderId,
+		P::TargetHeaderId,
+		P::Proof,
+		SourceNoncesRange = SC::NoncesRange,
+		ProofParameters = SC::ProofParameters,
+		TargetNoncesData = TC::TargetNoncesData,
+	>,
+) -> Result<(), FailedClient> {
+	let mut progress_context = Instant::now();
+	let mut race_state = RaceState::default();
+	let mut stall_countdown = Instant::now();
+
+	let mut source_retry_backoff = retry_backoff();
+	let mut source_client_is_online = true;
+	let mut source_nonces_required = false;
+	let source_nonces = futures::future::Fuse::terminated();
+	let source_generate_proof = futures::future::Fuse::terminated();
+	let source_go_offline_future = futures::future::Fuse::terminated();
+
+	let mut target_retry_backoff = retry_backoff();
+	let mut target_client_is_online = true;
+	let mut target_best_nonces_required = false;
+	let mut target_finalized_nonces_required = false;
+	let target_best_nonces = futures::future::Fuse::terminated();
+	let target_finalized_nonces = futures::future::Fuse::terminated();
+	let target_submit_proof = futures::future::Fuse::terminated();
+	let target_go_offline_future = futures::future::Fuse::terminated();
+
+	futures::pin_mut!(
+		race_source_updated,
+		source_nonces,
+		source_generate_proof,
+		source_go_offline_future,
+		race_target_updated,
+		target_best_nonces,
+		target_finalized_nonces,
+		target_submit_proof,
+		target_go_offline_future,
+	);
+
+	loop {
+		futures::select! {
+			// when headers ids are updated
+			source_state = race_source_updated.next() => {
+				if let Some(source_state) = source_state {
+					let is_source_state_updated = race_state.best_finalized_source_header_id_at_source.as_ref()
+						!= Some(&source_state.best_finalized_self);
+					if is_source_state_updated {
+						source_nonces_required = true;
+						race_state.best_finalized_source_header_id_at_source = Some(source_state.best_finalized_self);
+					}
+				}
+			},
+			target_state = race_target_updated.next() => {
+				if let Some(target_state) = target_state {
+					let is_target_best_state_updated = race_state.best_target_header_id.as_ref()
+						!= Some(&target_state.best_self);
+
+					if is_target_best_state_updated {
+						target_best_nonces_required = true;
+						race_state.best_target_header_id = Some(target_state.best_self);
+						race_state.best_finalized_source_header_id_at_best_target
+							= Some(target_state.best_finalized_peer_at_best_self);
+					}
+
+					let is_target_finalized_state_updated = race_state.best_finalized_target_header_id.as_ref()
+						!= Some(&target_state.best_finalized_self);
+					if  is_target_finalized_state_updated {
+						target_finalized_nonces_required = true;
+						race_state.best_finalized_target_header_id = Some(target_state.best_finalized_self);
+					}
+				}
+			},
+
+			// when nonces are updated
+			nonces = source_nonces => {
+				source_nonces_required = false;
+
+				source_client_is_online = process_future_result(
+					nonces,
+					&mut source_retry_backoff,
+					|(at_block, nonces)| {
+						log::debug!(
+							target: "bridge",
+							"Received nonces from {}: {:?}",
+							P::source_name(),
+							nonces,
+						);
+
+						strategy.source_nonces_updated(at_block, nonces);
+					},
+					&mut source_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving nonces from {}", P::source_name()),
+				).fail_if_connection_error(FailedClient::Source)?;
+			},
+			nonces = target_best_nonces => {
+				target_best_nonces_required = false;
+
+				target_client_is_online = process_future_result(
+					nonces,
+					&mut target_retry_backoff,
+					|(_, nonces)| {
+						log::debug!(
+							target: "bridge",
+							"Received best nonces from {}: {:?}",
+							P::target_name(),
+							nonces,
+						);
+
+						let prev_best_at_target = strategy.best_at_target();
+						strategy.best_target_nonces_updated(nonces, &mut race_state);
+						if strategy.best_at_target() != prev_best_at_target {
+							stall_countdown = Instant::now();
+						}
+					},
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving best nonces from {}", P::target_name()),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+			nonces = target_finalized_nonces => {
+				target_finalized_nonces_required = false;
+
+				target_client_is_online = process_future_result(
+					nonces,
+					&mut target_retry_backoff,
+					|(_, nonces)| {
+						log::debug!(
+							target: "bridge",
+							"Received finalized nonces from {}: {:?}",
+							P::target_name(),
+							nonces,
+						);
+
+						strategy.finalized_target_nonces_updated(nonces, &mut race_state);
+					},
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving finalized nonces from {}", P::target_name()),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+
+			// proof generation and submission
+			proof = source_generate_proof => {
+				source_client_is_online = process_future_result(
+					proof,
+					&mut source_retry_backoff,
+					|(at_block, nonces_range, proof)| {
+						log::debug!(
+							target: "bridge",
+							"Received proof for nonces in range {:?} from {}",
+							nonces_range,
+							P::source_name(),
+						);
+
+						race_state.nonces_to_submit = Some((at_block, nonces_range, proof));
+					},
+					&mut source_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error generating proof at {}", P::source_name()),
+				).fail_if_connection_error(FailedClient::Source)?;
+			},
+			proof_submit_result = target_submit_proof => {
+				target_client_is_online = process_future_result(
+					proof_submit_result,
+					&mut target_retry_backoff,
+					|nonces_range| {
+						log::debug!(
+							target: "bridge",
+							"Successfully submitted proof of nonces {:?} to {}",
+							nonces_range,
+							P::target_name(),
+						);
+
+						race_state.nonces_to_submit = None;
+						race_state.nonces_submitted = Some(nonces_range);
+						stall_countdown = Instant::now();
+					},
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error submitting proof {}", P::target_name()),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+
+			// when we're ready to retry request
+			_ = source_go_offline_future => {
+				source_client_is_online = true;
+			},
+			_ = target_go_offline_future => {
+				target_client_is_online = true;
+			},
+		}
+
+		progress_context = print_race_progress::<P, _>(progress_context, &strategy);
+
+		if stall_countdown.elapsed() > stall_timeout {
+			log::warn!(
+				target: "bridge",
+				"{} -> {} race has stalled. State: {:?}. Strategy: {:?}",
+				P::source_name(),
+				P::target_name(),
+				race_state,
+				strategy,
+			);
+
+			return Err(FailedClient::Both);
+		} else if race_state.nonces_to_submit.is_none() && race_state.nonces_submitted.is_none() && strategy.is_empty()
+		{
+			stall_countdown = Instant::now();
+		}
+
+		if source_client_is_online {
+			source_client_is_online = false;
+
+			let nonces_to_deliver = select_nonces_to_deliver(&race_state, &mut strategy);
+			let best_at_source = strategy.best_at_source();
+
+			if let Some((at_block, nonces_range, proof_parameters)) = nonces_to_deliver {
+				log::debug!(
+					target: "bridge",
+					"Asking {} to prove nonces in range {:?} at block {:?}",
+					P::source_name(),
+					nonces_range,
+					at_block,
+				);
+				source_generate_proof.set(
+					race_source
+						.generate_proof(at_block, nonces_range, proof_parameters)
+						.fuse(),
+				);
+			} else if source_nonces_required && best_at_source.is_some() {
+				log::debug!(target: "bridge", "Asking {} about message nonces", P::source_name());
+				let at_block = race_state
+					.best_finalized_source_header_id_at_source
+					.as_ref()
+					.expect(
+						"source_nonces_required is only true when\
+						best_finalized_source_header_id_at_source is Some; qed",
+					)
+					.clone();
+				source_nonces.set(
+					race_source
+						.nonces(at_block, best_at_source.expect("guaranteed by if condition; qed"))
+						.fuse(),
+				);
+			} else {
+				source_client_is_online = true;
+			}
+		}
+
+		if target_client_is_online {
+			target_client_is_online = false;
+
+			if let Some((at_block, nonces_range, proof)) = race_state.nonces_to_submit.as_ref() {
+				log::debug!(
+					target: "bridge",
+					"Going to submit proof of messages in range {:?} to {} node",
+					nonces_range,
+					P::target_name(),
+				);
+				target_submit_proof.set(
+					race_target
+						.submit_proof(at_block.clone(), nonces_range.clone(), proof.clone())
+						.fuse(),
+				);
+			} else if target_best_nonces_required {
+				log::debug!(target: "bridge", "Asking {} about best message nonces", P::target_name());
+				let at_block = race_state
+					.best_target_header_id
+					.as_ref()
+					.expect("target_best_nonces_required is only true when best_target_header_id is Some; qed")
+					.clone();
+				target_best_nonces.set(race_target.nonces(at_block, false).fuse());
+			} else if target_finalized_nonces_required {
+				log::debug!(target: "bridge", "Asking {} about finalized message nonces", P::target_name());
+				let at_block = race_state
+					.best_finalized_target_header_id
+					.as_ref()
+					.expect(
+						"target_finalized_nonces_required is only true when\
+						best_finalized_target_header_id is Some; qed",
+					)
+					.clone();
+				target_finalized_nonces.set(race_target.nonces(at_block, true).fuse());
+			} else {
+				target_client_is_online = true;
+			}
+		}
+	}
+}
+
+impl<SourceHeaderId, TargetHeaderId, Proof> Default for RaceState<SourceHeaderId, TargetHeaderId, Proof> {
+	fn default() -> Self {
+		RaceState {
+			best_finalized_source_header_id_at_source: None,
+			best_finalized_source_header_id_at_best_target: None,
+			best_target_header_id: None,
+			best_finalized_target_header_id: None,
+			nonces_to_submit: None,
+			nonces_submitted: None,
+		}
+	}
+}
+
+/// Print race progress.
+fn print_race_progress<P, S>(prev_time: Instant, strategy: &S) -> Instant
+where
+	P: MessageRace,
+	S: RaceStrategy<P::SourceHeaderId, P::TargetHeaderId, P::Proof>,
+{
+	let now_time = Instant::now();
+
+	let need_update = now_time.saturating_duration_since(prev_time) > Duration::from_secs(10);
+	if !need_update {
+		return prev_time;
+	}
+
+	let now_best_nonce_at_source = strategy.best_at_source();
+	let now_best_nonce_at_target = strategy.best_at_target();
+	log::info!(
+		target: "bridge",
+		"Synced {:?} of {:?} nonces in {} -> {} race",
+		now_best_nonce_at_target,
+		now_best_nonce_at_source,
+		P::source_name(),
+		P::target_name(),
+	);
+	now_time
+}
+
+fn select_nonces_to_deliver<SourceHeaderId, TargetHeaderId, Proof, Strategy>(
+	race_state: &RaceState<SourceHeaderId, TargetHeaderId, Proof>,
+	strategy: &mut Strategy,
+) -> Option<(SourceHeaderId, RangeInclusive<MessageNonce>, Strategy::ProofParameters)>
+where
+	SourceHeaderId: Clone,
+	Strategy: RaceStrategy<SourceHeaderId, TargetHeaderId, Proof>,
+{
+	race_state
+		.best_finalized_source_header_id_at_best_target
+		.as_ref()
+		.and_then(|best_finalized_source_header_id_at_best_target| {
+			strategy
+				.select_nonces_to_deliver(&race_state)
+				.map(|(nonces_range, proof_parameters)| {
+					(
+						best_finalized_source_header_id_at_best_target.clone(),
+						nonces_range,
+						proof_parameters,
+					)
+				})
+		})
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::message_race_strategy::BasicStrategy;
+	use relay_utils::HeaderId;
+
+	#[test]
+	fn proof_is_generated_at_best_block_known_to_target_node() {
+		const GENERATED_AT: u64 = 6;
+		const BEST_AT_SOURCE: u64 = 10;
+		const BEST_AT_TARGET: u64 = 8;
+
+		// target node only knows about source' BEST_AT_TARGET block
+		// source node has BEST_AT_SOURCE > BEST_AT_TARGET block
+		let mut race_state = RaceState::<_, _, ()> {
+			best_finalized_source_header_id_at_source: Some(HeaderId(BEST_AT_SOURCE, BEST_AT_SOURCE)),
+			best_finalized_source_header_id_at_best_target: Some(HeaderId(BEST_AT_TARGET, BEST_AT_TARGET)),
+			best_target_header_id: Some(HeaderId(0, 0)),
+			best_finalized_target_header_id: Some(HeaderId(0, 0)),
+			nonces_to_submit: None,
+			nonces_submitted: None,
+		};
+
+		// we have some nonces to deliver and they're generated at GENERATED_AT < BEST_AT_SOURCE
+		let mut strategy = BasicStrategy::new();
+		strategy.source_nonces_updated(
+			HeaderId(GENERATED_AT, GENERATED_AT),
+			SourceClientNonces {
+				new_nonces: 0..=10,
+				confirmed_nonce: None,
+			},
+		);
+		strategy.best_target_nonces_updated(
+			TargetClientNonces {
+				latest_nonce: 5u64,
+				nonces_data: (),
+			},
+			&mut race_state,
+		);
+
+		// the proof will be generated on source, but using BEST_AT_TARGET block
+		assert_eq!(
+			select_nonces_to_deliver(&race_state, &mut strategy),
+			Some((HeaderId(BEST_AT_TARGET, BEST_AT_TARGET), 6..=10, (),))
+		);
+	}
+}

bridges/relays/generic/messages/src/message_race_receiving.rs

@@ -0,0 +1,232 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+//! Message receiving race delivers proof-of-messages-delivery from lane.target to lane.source.
+
+use crate::message_lane::{MessageLane, SourceHeaderIdOf, TargetHeaderIdOf};
+use crate::message_lane_loop::{
+	SourceClient as MessageLaneSourceClient, SourceClientState, TargetClient as MessageLaneTargetClient,
+	TargetClientState,
+};
+use crate::message_race_loop::{
+	MessageRace, NoncesRange, SourceClient, SourceClientNonces, TargetClient, TargetClientNonces,
+};
+use crate::message_race_strategy::BasicStrategy;
+use crate::metrics::MessageLaneLoopMetrics;
+
+use async_trait::async_trait;
+use bp_message_lane::MessageNonce;
+use futures::stream::FusedStream;
+use relay_utils::FailedClient;
+use std::{marker::PhantomData, ops::RangeInclusive, time::Duration};
+
+/// Message receiving confirmations delivery strategy.
+type ReceivingConfirmationsBasicStrategy<P> = BasicStrategy<
+	<P as MessageLane>::TargetHeaderNumber,
+	<P as MessageLane>::TargetHeaderHash,
+	<P as MessageLane>::SourceHeaderNumber,
+	<P as MessageLane>::SourceHeaderHash,
+	RangeInclusive<MessageNonce>,
+	<P as MessageLane>::MessagesReceivingProof,
+>;
+
+/// Run receiving confirmations race.
+pub async fn run<P: MessageLane>(
+	source_client: impl MessageLaneSourceClient<P>,
+	source_state_updates: impl FusedStream<Item = SourceClientState<P>>,
+	target_client: impl MessageLaneTargetClient<P>,
+	target_state_updates: impl FusedStream<Item = TargetClientState<P>>,
+	stall_timeout: Duration,
+	metrics_msg: Option<MessageLaneLoopMetrics>,
+) -> Result<(), FailedClient> {
+	crate::message_race_loop::run(
+		ReceivingConfirmationsRaceSource {
+			client: target_client,
+			metrics_msg: metrics_msg.clone(),
+			_phantom: Default::default(),
+		},
+		target_state_updates,
+		ReceivingConfirmationsRaceTarget {
+			client: source_client,
+			metrics_msg,
+			_phantom: Default::default(),
+		},
+		source_state_updates,
+		stall_timeout,
+		ReceivingConfirmationsBasicStrategy::<P>::new(),
+	)
+	.await
+}
+
+/// Messages receiving confirmations race.
+struct ReceivingConfirmationsRace<P>(std::marker::PhantomData<P>);
+
+impl<P: MessageLane> MessageRace for ReceivingConfirmationsRace<P> {
+	type SourceHeaderId = TargetHeaderIdOf<P>;
+	type TargetHeaderId = SourceHeaderIdOf<P>;
+
+	type MessageNonce = MessageNonce;
+	type Proof = P::MessagesReceivingProof;
+
+	fn source_name() -> String {
+		format!("{}::ReceivingConfirmationsDelivery", P::TARGET_NAME)
+	}
+
+	fn target_name() -> String {
+		format!("{}::ReceivingConfirmationsDelivery", P::SOURCE_NAME)
+	}
+}
+
+/// Message receiving confirmations race source, which is a target of the lane.
+struct ReceivingConfirmationsRaceSource<P: MessageLane, C> {
+	client: C,
+	metrics_msg: Option<MessageLaneLoopMetrics>,
+	_phantom: PhantomData<P>,
+}
+
+#[async_trait]
+impl<P, C> SourceClient<ReceivingConfirmationsRace<P>> for ReceivingConfirmationsRaceSource<P, C>
+where
+	P: MessageLane,
+	C: MessageLaneTargetClient<P>,
+{
+	type Error = C::Error;
+	type NoncesRange = RangeInclusive<MessageNonce>;
+	type ProofParameters = ();
+
+	async fn nonces(
+		&self,
+		at_block: TargetHeaderIdOf<P>,
+		prev_latest_nonce: MessageNonce,
+	) -> Result<(TargetHeaderIdOf<P>, SourceClientNonces<Self::NoncesRange>), Self::Error> {
+		let (at_block, latest_received_nonce) = self.client.latest_received_nonce(at_block).await?;
+		if let Some(metrics_msg) = self.metrics_msg.as_ref() {
+			metrics_msg.update_target_latest_received_nonce::<P>(latest_received_nonce);
+		}
+		Ok((
+			at_block,
+			SourceClientNonces {
+				new_nonces: prev_latest_nonce + 1..=latest_received_nonce,
+				confirmed_nonce: None,
+			},
+		))
+	}
+
+	#[allow(clippy::unit_arg)]
+	async fn generate_proof(
+		&self,
+		at_block: TargetHeaderIdOf<P>,
+		nonces: RangeInclusive<MessageNonce>,
+		_proof_parameters: Self::ProofParameters,
+	) -> Result<
+		(
+			TargetHeaderIdOf<P>,
+			RangeInclusive<MessageNonce>,
+			P::MessagesReceivingProof,
+		),
+		Self::Error,
+	> {
+		self.client
+			.prove_messages_receiving(at_block)
+			.await
+			.map(|(at_block, proof)| (at_block, nonces, proof))
+	}
+}
+
+/// Message receiving confirmations race target, which is a source of the lane.
+struct ReceivingConfirmationsRaceTarget<P: MessageLane, C> {
+	client: C,
+	metrics_msg: Option<MessageLaneLoopMetrics>,
+	_phantom: PhantomData<P>,
+}
+
+#[async_trait]
+impl<P, C> TargetClient<ReceivingConfirmationsRace<P>> for ReceivingConfirmationsRaceTarget<P, C>
+where
+	P: MessageLane,
+	C: MessageLaneSourceClient<P>,
+{
+	type Error = C::Error;
+	type TargetNoncesData = ();
+
+	async fn nonces(
+		&self,
+		at_block: SourceHeaderIdOf<P>,
+		update_metrics: bool,
+	) -> Result<(SourceHeaderIdOf<P>, TargetClientNonces<()>), Self::Error> {
+		let (at_block, latest_confirmed_nonce) = self.client.latest_confirmed_received_nonce(at_block).await?;
+		if update_metrics {
+			if let Some(metrics_msg) = self.metrics_msg.as_ref() {
+				metrics_msg.update_source_latest_confirmed_nonce::<P>(latest_confirmed_nonce);
+			}
+		}
+		Ok((
+			at_block,
+			TargetClientNonces {
+				latest_nonce: latest_confirmed_nonce,
+				nonces_data: (),
+			},
+		))
+	}
+
+	async fn submit_proof(
+		&self,
+		generated_at_block: TargetHeaderIdOf<P>,
+		nonces: RangeInclusive<MessageNonce>,
+		proof: P::MessagesReceivingProof,
+	) -> Result<RangeInclusive<MessageNonce>, Self::Error> {
+		self.client
+			.submit_messages_receiving_proof(generated_at_block, proof)
+			.await?;
+		Ok(nonces)
+	}
+}
+
+impl NoncesRange for RangeInclusive<MessageNonce> {
+	fn begin(&self) -> MessageNonce {
+		*RangeInclusive::<MessageNonce>::start(self)
+	}
+
+	fn end(&self) -> MessageNonce {
+		*RangeInclusive::<MessageNonce>::end(self)
+	}
+
+	fn greater_than(self, nonce: MessageNonce) -> Option<Self> {
+		let next_nonce = nonce + 1;
+		let end = *self.end();
+		if next_nonce > end {
+			None
+		} else {
+			Some(std::cmp::max(self.begin(), next_nonce)..=end)
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn range_inclusive_works_as_nonces_range() {
+		let range = 20..=30;
+
+		assert_eq!(NoncesRange::begin(&range), 20);
+		assert_eq!(NoncesRange::end(&range), 30);
+		assert_eq!(range.clone().greater_than(10), Some(20..=30));
+		assert_eq!(range.clone().greater_than(19), Some(20..=30));
+		assert_eq!(range.clone().greater_than(20), Some(21..=30));
+		assert_eq!(range.clone().greater_than(25), Some(26..=30));
+		assert_eq!(range.clone().greater_than(29), Some(30..=30));
+		assert_eq!(range.greater_than(30), None);
+	}
+}

bridges/relays/generic/messages/src/message_race_strategy.rs

@@ -0,0 +1,479 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+//! Basic delivery strategy. The strategy selects nonces if:
+//!
+//! 1) there are more nonces on the source side than on the target side;
+//! 2) new nonces may be proved to target node (i.e. they have appeared at the
+//!    block, which is known to the target node).
+
+use crate::message_race_loop::{NoncesRange, RaceState, RaceStrategy, SourceClientNonces, TargetClientNonces};
+
+use bp_message_lane::MessageNonce;
+use relay_utils::HeaderId;
+use std::{collections::VecDeque, fmt::Debug, marker::PhantomData, ops::RangeInclusive};
+
+/// Nonces delivery strategy.
+#[derive(Debug)]
+pub struct BasicStrategy<
+	SourceHeaderNumber,
+	SourceHeaderHash,
+	TargetHeaderNumber,
+	TargetHeaderHash,
+	SourceNoncesRange,
+	Proof,
+> {
+	/// All queued nonces.
+	source_queue: VecDeque<(HeaderId<SourceHeaderHash, SourceHeaderNumber>, SourceNoncesRange)>,
+	/// Best nonce known to target node (at its best block). `None` if it has not been received yet.
+	best_target_nonce: Option<MessageNonce>,
+	/// Unused generic types dump.
+	_phantom: PhantomData<(TargetHeaderNumber, TargetHeaderHash, Proof)>,
+}
+
+impl<SourceHeaderNumber, SourceHeaderHash, TargetHeaderNumber, TargetHeaderHash, SourceNoncesRange, Proof>
+	BasicStrategy<SourceHeaderNumber, SourceHeaderHash, TargetHeaderNumber, TargetHeaderHash, SourceNoncesRange, Proof>
+where
+	SourceHeaderHash: Clone,
+	SourceHeaderNumber: Clone + Ord,
+	SourceNoncesRange: NoncesRange,
+{
+	/// Create new delivery strategy.
+	pub fn new() -> Self {
+		BasicStrategy {
+			source_queue: VecDeque::new(),
+			best_target_nonce: None,
+			_phantom: Default::default(),
+		}
+	}
+
+	/// Mutable reference to source queue to use in tests.
+	#[cfg(test)]
+	pub(crate) fn source_queue_mut(
+		&mut self,
+	) -> &mut VecDeque<(HeaderId<SourceHeaderHash, SourceHeaderNumber>, SourceNoncesRange)> {
+		&mut self.source_queue
+	}
+
+	/// Should return `Some(nonces)` if we need to deliver proof of `nonces` (and associated
+	/// data) from source to target node.
+	///
+	/// The `selector` function receives range of nonces and should return `None` if the whole
+	/// range needs to be delivered. If there are some nonces in the range that can't be delivered
+	/// right now, it should return `Some` with 'undeliverable' nonces. Please keep in mind that
+	/// this should be the sub-range that the passed range ends with, because nonces are always
+	/// delivered in-order. Otherwise the function will panic.
+	pub fn select_nonces_to_deliver_with_selector(
+		&mut self,
+		race_state: &RaceState<
+			HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+			HeaderId<TargetHeaderHash, TargetHeaderNumber>,
+			Proof,
+		>,
+		mut selector: impl FnMut(SourceNoncesRange) -> Option<SourceNoncesRange>,
+	) -> Option<RangeInclusive<MessageNonce>> {
+		// if we do not know best nonce at target node, we can't select anything
+		let target_nonce = self.best_target_nonce?;
+
+		// if we have already selected nonces that we want to submit, do nothing
+		if race_state.nonces_to_submit.is_some() {
+			return None;
+		}
+
+		// if we already submitted some nonces, do nothing
+		if race_state.nonces_submitted.is_some() {
+			return None;
+		}
+
+		// 1) we want to deliver all nonces, starting from `target_nonce + 1`
+		// 2) we can't deliver new nonce until header, that has emitted this nonce, is finalized
+		// by target client
+		// 3) selector is used for more complicated logic
+		let best_header_at_target = &race_state.best_finalized_source_header_id_at_best_target.as_ref()?;
+		let mut nonces_end = None;
+		while let Some((queued_at, queued_range)) = self.source_queue.pop_front() {
+			// select (sub) range to deliver
+			let queued_range_begin = queued_range.begin();
+			let queued_range_end = queued_range.end();
+			let range_to_requeue = if queued_at.0 > best_header_at_target.0 {
+				// if header that has queued the range is not yet finalized at bridged chain,
+				// we can't prove anything
+				Some(queued_range)
+			} else {
+				// selector returns `Some(range)` if this `range` needs to be requeued
+				selector(queued_range)
+			};
+
+			// requeue (sub) range and update range to deliver
+			match range_to_requeue {
+				Some(range_to_requeue) => {
+					assert!(
+						range_to_requeue.begin() <= range_to_requeue.end()
+							&& range_to_requeue.begin() >= queued_range_begin
+							&& range_to_requeue.end() == queued_range_end,
+						"Incorrect implementation of internal `selector` function. Expected original\
+						range {:?} to end with returned range {:?}",
+						queued_range_begin..=queued_range_end,
+						range_to_requeue,
+					);
+
+					if range_to_requeue.begin() != queued_range_begin {
+						nonces_end = Some(range_to_requeue.begin() - 1);
+					}
+					self.source_queue.push_front((queued_at, range_to_requeue));
+					break;
+				}
+				None => {
+					nonces_end = Some(queued_range_end);
+				}
+			}
+		}
+
+		nonces_end.map(|nonces_end| RangeInclusive::new(target_nonce + 1, nonces_end))
+	}
+}
+
+impl<SourceHeaderNumber, SourceHeaderHash, TargetHeaderNumber, TargetHeaderHash, SourceNoncesRange, Proof>
+	RaceStrategy<HeaderId<SourceHeaderHash, SourceHeaderNumber>, HeaderId<TargetHeaderHash, TargetHeaderNumber>, Proof>
+	for BasicStrategy<SourceHeaderNumber, SourceHeaderHash, TargetHeaderNumber, TargetHeaderHash, SourceNoncesRange, Proof>
+where
+	SourceHeaderHash: Clone + Debug,
+	SourceHeaderNumber: Clone + Ord + Debug,
+	SourceNoncesRange: NoncesRange + Debug,
+	TargetHeaderHash: Debug,
+	TargetHeaderNumber: Debug,
+	Proof: Debug,
+{
+	type SourceNoncesRange = SourceNoncesRange;
+	type ProofParameters = ();
+	type TargetNoncesData = ();
+
+	fn is_empty(&self) -> bool {
+		self.source_queue.is_empty()
+	}
+
+	fn best_at_source(&self) -> Option<MessageNonce> {
+		let best_in_queue = self.source_queue.back().map(|(_, range)| range.end());
+		match (best_in_queue, self.best_target_nonce) {
+			(Some(best_in_queue), Some(best_target_nonce)) if best_in_queue > best_target_nonce => Some(best_in_queue),
+			(_, Some(best_target_nonce)) => Some(best_target_nonce),
+			(_, None) => None,
+		}
+	}
+
+	fn best_at_target(&self) -> Option<MessageNonce> {
+		self.best_target_nonce
+	}
+
+	fn source_nonces_updated(
+		&mut self,
+		at_block: HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+		nonces: SourceClientNonces<SourceNoncesRange>,
+	) {
+		let best_in_queue = self
+			.source_queue
+			.back()
+			.map(|(_, range)| range.end())
+			.or(self.best_target_nonce)
+			.unwrap_or_default();
+		self.source_queue.extend(
+			nonces
+				.new_nonces
+				.greater_than(best_in_queue)
+				.into_iter()
+				.map(move |range| (at_block.clone(), range)),
+		)
+	}
+
+	fn best_target_nonces_updated(
+		&mut self,
+		nonces: TargetClientNonces<()>,
+		race_state: &mut RaceState<
+			HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+			HeaderId<TargetHeaderHash, TargetHeaderNumber>,
+			Proof,
+		>,
+	) {
+		let nonce = nonces.latest_nonce;
+
+		if let Some(best_target_nonce) = self.best_target_nonce {
+			if nonce < best_target_nonce {
+				return;
+			}
+		}
+
+		while let Some(true) = self.source_queue.front().map(|(_, range)| range.begin() <= nonce) {
+			let maybe_subrange = self
+				.source_queue
+				.pop_front()
+				.and_then(|(at_block, range)| range.greater_than(nonce).map(|subrange| (at_block, subrange)));
+			if let Some((at_block, subrange)) = maybe_subrange {
+				self.source_queue.push_front((at_block, subrange));
+				break;
+			}
+		}
+
+		let need_to_select_new_nonces = race_state
+			.nonces_to_submit
+			.as_ref()
+			.map(|(_, nonces, _)| *nonces.end() <= nonce)
+			.unwrap_or(false);
+		if need_to_select_new_nonces {
+			race_state.nonces_to_submit = None;
+		}
+
+		let need_new_nonces_to_submit = race_state
+			.nonces_submitted
+			.as_ref()
+			.map(|nonces| *nonces.end() <= nonce)
+			.unwrap_or(false);
+		if need_new_nonces_to_submit {
+			race_state.nonces_submitted = None;
+		}
+
+		self.best_target_nonce = Some(std::cmp::max(
+			self.best_target_nonce.unwrap_or(nonces.latest_nonce),
+			nonce,
+		));
+	}
+
+	fn finalized_target_nonces_updated(
+		&mut self,
+		nonces: TargetClientNonces<()>,
+		_race_state: &mut RaceState<
+			HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+			HeaderId<TargetHeaderHash, TargetHeaderNumber>,
+			Proof,
+		>,
+	) {
+		self.best_target_nonce = Some(std::cmp::max(
+			self.best_target_nonce.unwrap_or(nonces.latest_nonce),
+			nonces.latest_nonce,
+		));
+	}
+
+	fn select_nonces_to_deliver(
+		&mut self,
+		race_state: &RaceState<
+			HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+			HeaderId<TargetHeaderHash, TargetHeaderNumber>,
+			Proof,
+		>,
+	) -> Option<(RangeInclusive<MessageNonce>, Self::ProofParameters)> {
+		self.select_nonces_to_deliver_with_selector(race_state, |_| None)
+			.map(|range| (range, ()))
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::message_lane::MessageLane;
+	use crate::message_lane_loop::tests::{header_id, TestMessageLane, TestMessagesProof};
+
+	type SourceNoncesRange = RangeInclusive<MessageNonce>;
+
+	type BasicStrategy<P> = super::BasicStrategy<
+		<P as MessageLane>::SourceHeaderNumber,
+		<P as MessageLane>::SourceHeaderHash,
+		<P as MessageLane>::TargetHeaderNumber,
+		<P as MessageLane>::TargetHeaderHash,
+		SourceNoncesRange,
+		<P as MessageLane>::MessagesProof,
+	>;
+
+	fn source_nonces(new_nonces: SourceNoncesRange) -> SourceClientNonces<SourceNoncesRange> {
+		SourceClientNonces {
+			new_nonces,
+			confirmed_nonce: None,
+		}
+	}
+
+	fn target_nonces(latest_nonce: MessageNonce) -> TargetClientNonces<()> {
+		TargetClientNonces {
+			latest_nonce,
+			nonces_data: (),
+		}
+	}
+
+	#[test]
+	fn strategy_is_empty_works() {
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		assert_eq!(strategy.is_empty(), true);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=1));
+		assert_eq!(strategy.is_empty(), false);
+	}
+
+	#[test]
+	fn best_at_source_is_never_lower_than_target_nonce() {
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		assert_eq!(strategy.best_at_source(), None);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=5));
+		assert_eq!(strategy.best_at_source(), None);
+		strategy.best_target_nonces_updated(target_nonces(10), &mut Default::default());
+		assert_eq!(strategy.source_queue, vec![]);
+		assert_eq!(strategy.best_at_source(), Some(10));
+	}
+
+	#[test]
+	fn source_nonce_is_never_lower_than_known_target_nonce() {
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.best_target_nonces_updated(target_nonces(10), &mut Default::default());
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=5));
+		assert_eq!(strategy.source_queue, vec![]);
+	}
+
+	#[test]
+	fn source_nonce_is_never_lower_than_latest_known_source_nonce() {
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=5));
+		strategy.source_nonces_updated(header_id(2), source_nonces(1..=3));
+		strategy.source_nonces_updated(header_id(2), source_nonces(1..=5));
+		assert_eq!(strategy.source_queue, vec![(header_id(1), 1..=5)]);
+	}
+
+	#[test]
+	fn target_nonce_is_never_lower_than_latest_known_target_nonce() {
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		assert_eq!(strategy.best_target_nonce, None);
+		strategy.best_target_nonces_updated(target_nonces(10), &mut Default::default());
+		assert_eq!(strategy.best_target_nonce, Some(10));
+		strategy.best_target_nonces_updated(target_nonces(5), &mut Default::default());
+		assert_eq!(strategy.best_target_nonce, Some(10));
+	}
+
+	#[test]
+	fn updated_target_nonce_removes_queued_entries() {
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=5));
+		strategy.source_nonces_updated(header_id(2), source_nonces(6..=10));
+		strategy.source_nonces_updated(header_id(3), source_nonces(11..=15));
+		strategy.source_nonces_updated(header_id(4), source_nonces(16..=20));
+		strategy.best_target_nonces_updated(target_nonces(15), &mut Default::default());
+		assert_eq!(strategy.source_queue, vec![(header_id(4), 16..=20)]);
+		strategy.best_target_nonces_updated(target_nonces(17), &mut Default::default());
+		assert_eq!(strategy.source_queue, vec![(header_id(4), 18..=20)]);
+	}
+
+	#[test]
+	fn selected_nonces_are_dropped_on_target_nonce_update() {
+		let mut state = RaceState::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		state.nonces_to_submit = Some((header_id(1), 5..=10, (5..=10, None)));
+		strategy.best_target_nonces_updated(target_nonces(7), &mut state);
+		assert!(state.nonces_to_submit.is_some());
+		strategy.best_target_nonces_updated(target_nonces(10), &mut state);
+		assert!(state.nonces_to_submit.is_none());
+	}
+
+	#[test]
+	fn submitted_nonces_are_dropped_on_target_nonce_update() {
+		let mut state = RaceState::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		state.nonces_submitted = Some(5..=10);
+		strategy.best_target_nonces_updated(target_nonces(7), &mut state);
+		assert!(state.nonces_submitted.is_some());
+		strategy.best_target_nonces_updated(target_nonces(10), &mut state);
+		assert!(state.nonces_submitted.is_none());
+	}
+
+	#[test]
+	fn nothing_is_selected_if_something_is_already_selected() {
+		let mut state = RaceState::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		state.nonces_to_submit = Some((header_id(1), 1..=10, (1..=10, None)));
+		strategy.best_target_nonces_updated(target_nonces(0), &mut state);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=10));
+		assert_eq!(strategy.select_nonces_to_deliver(&state), None);
+	}
+
+	#[test]
+	fn nothing_is_selected_if_something_is_already_submitted() {
+		let mut state = RaceState::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		state.nonces_submitted = Some(1..=10);
+		strategy.best_target_nonces_updated(target_nonces(0), &mut state);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=10));
+		assert_eq!(strategy.select_nonces_to_deliver(&state), None);
+	}
+
+	#[test]
+	fn select_nonces_to_deliver_works() {
+		let mut state = RaceState::<_, _, TestMessagesProof>::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.best_target_nonces_updated(target_nonces(0), &mut state);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=1));
+		strategy.source_nonces_updated(header_id(2), source_nonces(2..=2));
+		strategy.source_nonces_updated(header_id(3), source_nonces(3..=6));
+		strategy.source_nonces_updated(header_id(5), source_nonces(7..=8));
+
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(4));
+		assert_eq!(strategy.select_nonces_to_deliver(&state), Some((1..=6, ())));
+		strategy.best_target_nonces_updated(target_nonces(6), &mut state);
+		assert_eq!(strategy.select_nonces_to_deliver(&state), None);
+
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(5));
+		assert_eq!(strategy.select_nonces_to_deliver(&state), Some((7..=8, ())));
+		strategy.best_target_nonces_updated(target_nonces(8), &mut state);
+		assert_eq!(strategy.select_nonces_to_deliver(&state), None);
+	}
+
+	#[test]
+	fn select_nonces_to_deliver_able_to_split_ranges_with_selector() {
+		let mut state = RaceState::<_, _, TestMessagesProof>::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.best_target_nonces_updated(target_nonces(0), &mut state);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=100));
+
+		state.best_finalized_source_header_id_at_source = Some(header_id(1));
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(1));
+		state.best_target_header_id = Some(header_id(1));
+
+		assert_eq!(
+			strategy.select_nonces_to_deliver_with_selector(&state, |_| Some(50..=100)),
+			Some(1..=49),
+		);
+	}
+
+	fn run_panic_test_for_incorrect_selector(
+		invalid_selector: impl Fn(SourceNoncesRange) -> Option<SourceNoncesRange>,
+	) {
+		let mut state = RaceState::<_, _, TestMessagesProof>::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=100));
+		strategy.best_target_nonces_updated(target_nonces(50), &mut state);
+		state.best_finalized_source_header_id_at_source = Some(header_id(1));
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(1));
+		state.best_target_header_id = Some(header_id(1));
+		strategy.select_nonces_to_deliver_with_selector(&state, invalid_selector);
+	}
+
+	#[test]
+	#[should_panic]
+	fn select_nonces_to_deliver_panics_if_selector_returns_empty_range() {
+		#[allow(clippy::reversed_empty_ranges)]
+		run_panic_test_for_incorrect_selector(|_| Some(2..=1))
+	}
+
+	#[test]
+	#[should_panic]
+	fn select_nonces_to_deliver_panics_if_selector_returns_range_that_starts_before_passed_range() {
+		run_panic_test_for_incorrect_selector(|range| Some(range.begin() - 1..=*range.end()))
+	}
+
+	#[test]
+	#[should_panic]
+	fn select_nonces_to_deliver_panics_if_selector_returns_range_with_mismatched_end() {
+		run_panic_test_for_incorrect_selector(|range| Some(range.begin()..=*range.end() + 1))
+	}
+}

bridges/relays/generic/messages/src/metrics.rs

@@ -0,0 +1,107 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Metrics for message lane relay loop.
+
+use crate::message_lane::MessageLane;
+use crate::message_lane_loop::{SourceClientState, TargetClientState};
+
+use bp_message_lane::MessageNonce;
+use relay_utils::metrics::{register, GaugeVec, Metrics, Opts, Registry, U64};
+
+/// Message lane relay metrics.
+///
+/// Cloning only clones references.
+#[derive(Clone)]
+pub struct MessageLaneLoopMetrics {
+	/// Best finalized block numbers - "source", "target", "source_at_target", "target_at_source".
+	best_block_numbers: GaugeVec<U64>,
+	/// Lane state nonces: "source_latest_generated", "source_latest_confirmed",
+	/// "target_latest_received", "target_latest_confirmed".
+	lane_state_nonces: GaugeVec<U64>,
+}
+
+impl Metrics for MessageLaneLoopMetrics {
+	fn register(&self, registry: &Registry) -> Result<(), String> {
+		register(self.best_block_numbers.clone(), registry).map_err(|e| e.to_string())?;
+		register(self.lane_state_nonces.clone(), registry).map_err(|e| e.to_string())?;
+		Ok(())
+	}
+}
+
+impl Default for MessageLaneLoopMetrics {
+	fn default() -> Self {
+		MessageLaneLoopMetrics {
+			best_block_numbers: GaugeVec::new(
+				Opts::new("best_block_numbers", "Best finalized block numbers"),
+				&["type"],
+			)
+			.expect("metric is static and thus valid; qed"),
+			lane_state_nonces: GaugeVec::new(Opts::new("lane_state_nonces", "Nonces of the lane state"), &["type"])
+				.expect("metric is static and thus valid; qed"),
+		}
+	}
+}
+
+impl MessageLaneLoopMetrics {
+	/// Update source client state metrics.
+	pub fn update_source_state<P: MessageLane>(&self, source_client_state: SourceClientState<P>) {
+		self.best_block_numbers
+			.with_label_values(&["source"])
+			.set(source_client_state.best_self.0.into());
+		self.best_block_numbers
+			.with_label_values(&["target_at_source"])
+			.set(source_client_state.best_finalized_peer_at_best_self.0.into());
+	}
+
+	/// Update target client state metrics.
+	pub fn update_target_state<P: MessageLane>(&self, target_client_state: TargetClientState<P>) {
+		self.best_block_numbers
+			.with_label_values(&["target"])
+			.set(target_client_state.best_self.0.into());
+		self.best_block_numbers
+			.with_label_values(&["source_at_target"])
+			.set(target_client_state.best_finalized_peer_at_best_self.0.into());
+	}
+
+	/// Update latest generated nonce at source.
+	pub fn update_source_latest_generated_nonce<P: MessageLane>(&self, source_latest_generated_nonce: MessageNonce) {
+		self.lane_state_nonces
+			.with_label_values(&["source_latest_generated"])
+			.set(source_latest_generated_nonce);
+	}
+
+	/// Update latest confirmed nonce at source.
+	pub fn update_source_latest_confirmed_nonce<P: MessageLane>(&self, source_latest_confirmed_nonce: MessageNonce) {
+		self.lane_state_nonces
+			.with_label_values(&["source_latest_confirmed"])
+			.set(source_latest_confirmed_nonce);
+	}
+
+	/// Update latest received nonce at target.
+	pub fn update_target_latest_received_nonce<P: MessageLane>(&self, target_latest_generated_nonce: MessageNonce) {
+		self.lane_state_nonces
+			.with_label_values(&["target_latest_received"])
+			.set(target_latest_generated_nonce);
+	}
+
+	/// Update latest confirmed nonce at target.
+	pub fn update_target_latest_confirmed_nonce<P: MessageLane>(&self, target_latest_confirmed_nonce: MessageNonce) {
+		self.lane_state_nonces
+			.with_label_values(&["target_latest_confirmed"])
+			.set(target_latest_confirmed_nonce);
+	}
+}

bridges/relays/generic/utils/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "relay-utils"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+
+[dependencies]
+ansi_term = "0.12"
+async-std = "1.6.5"
+async-trait = "0.1.40"
+backoff = "0.2"
+env_logger = "0.8.2"
+futures = "0.3.5"
+log = "0.4.11"
+num-traits = "0.2"
+sysinfo = "0.15"
+time = "0.2"
+
+# Substrate dependencies
+
+substrate-prometheus-endpoint = { git = "https://github.com/paritytech/substrate", branch = "master" }

bridges/relays/generic/utils/src/initialize.rs

@@ -0,0 +1,59 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Relayer initialization functions.
+
+use std::io::Write;
+
+/// Initialize relay environment.
+pub fn initialize_relay() {
+	let mut builder = env_logger::Builder::new();
+
+	let filters = match std::env::var("RUST_LOG") {
+		Ok(env_filters) => format!("bridge=info,{}", env_filters),
+		Err(_) => "bridge=info".into(),
+	};
+
+	builder.parse_filters(&filters);
+	builder.format(move |buf, record| {
+		writeln!(buf, "{}", {
+			let timestamp = time::OffsetDateTime::try_now_local()
+				.unwrap_or_else(|_| time::OffsetDateTime::now_utc())
+				.format("%Y-%m-%d %H:%M:%S %z");
+			if cfg!(windows) {
+				format!("{} {} {} {}", timestamp, record.level(), record.target(), record.args())
+			} else {
+				use ansi_term::Colour as Color;
+				let log_level = match record.level() {
+					log::Level::Error => Color::Fixed(9).bold().paint(record.level().to_string()),
+					log::Level::Warn => Color::Fixed(11).bold().paint(record.level().to_string()),
+					log::Level::Info => Color::Fixed(10).paint(record.level().to_string()),
+					log::Level::Debug => Color::Fixed(14).paint(record.level().to_string()),
+					log::Level::Trace => Color::Fixed(12).paint(record.level().to_string()),
+				};
+				format!(
+					"{} {} {} {}",
+					Color::Fixed(8).bold().paint(timestamp),
+					log_level,
+					Color::Fixed(8).paint(record.target()),
+					record.args()
+				)
+			}
+		})
+	});
+
+	builder.init();
+}

bridges/relays/generic/utils/src/lib.rs

@@ -0,0 +1,275 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Utilities used by different relays.
+
+use backoff::{backoff::Backoff, ExponentialBackoff};
+use futures::future::FutureExt;
+use std::time::Duration;
+
+/// Max delay after connection-unrelated error happened before we'll try the
+/// same request again.
+pub const MAX_BACKOFF_INTERVAL: Duration = Duration::from_secs(60);
+/// Delay after connection-related error happened before we'll try
+/// reconnection again.
+pub const CONNECTION_ERROR_DELAY: Duration = Duration::from_secs(10);
+
+pub mod initialize;
+pub mod metrics;
+pub mod relay_loop;
+
+/// Block number traits shared by all chains that relay is able to serve.
+pub trait BlockNumberBase:
+	'static
+	+ From<u32>
+	+ Into<u64>
+	+ Ord
+	+ Clone
+	+ Copy
+	+ Default
+	+ Send
+	+ Sync
+	+ std::fmt::Debug
+	+ std::fmt::Display
+	+ std::hash::Hash
+	+ std::ops::Add<Output = Self>
+	+ std::ops::Sub<Output = Self>
+	+ num_traits::CheckedSub
+	+ num_traits::Saturating
+	+ num_traits::Zero
+	+ num_traits::One
+{
+}
+
+impl<T> BlockNumberBase for T where
+	T: 'static
+		+ From<u32>
+		+ Into<u64>
+		+ Ord
+		+ Clone
+		+ Copy
+		+ Default
+		+ Send
+		+ Sync
+		+ std::fmt::Debug
+		+ std::fmt::Display
+		+ std::hash::Hash
+		+ std::ops::Add<Output = Self>
+		+ std::ops::Sub<Output = Self>
+		+ num_traits::CheckedSub
+		+ num_traits::Saturating
+		+ num_traits::Zero
+		+ num_traits::One
+{
+}
+
+/// Macro that returns (client, Err(error)) tuple from function if result is Err(error).
+#[macro_export]
+macro_rules! bail_on_error {
+	($result: expr) => {
+		match $result {
+			(client, Ok(result)) => (client, result),
+			(client, Err(error)) => return (client, Err(error)),
+		}
+	};
+}
+
+/// Macro that returns (client, Err(error)) tuple from function if result is Err(error).
+#[macro_export]
+macro_rules! bail_on_arg_error {
+	($result: expr, $client: ident) => {
+		match $result {
+			Ok(result) => result,
+			Err(error) => return ($client, Err(error)),
+		}
+	};
+}
+
+/// Ethereum header Id.
+#[derive(Debug, Default, Clone, Copy, Eq, Hash, PartialEq)]
+pub struct HeaderId<Hash, Number>(pub Number, pub Hash);
+
+/// Error type that can signal connection errors.
+pub trait MaybeConnectionError {
+	/// Returns true if error (maybe) represents connection error.
+	fn is_connection_error(&self) -> bool;
+}
+
+/// Stringified error that may be either connection-related or not.
+#[derive(Debug)]
+pub enum StringifiedMaybeConnectionError {
+	/// The error is connection-related error.
+	Connection(String),
+	/// The error is connection-unrelated error.
+	NonConnection(String),
+}
+
+impl StringifiedMaybeConnectionError {
+	/// Create new stringified connection error.
+	pub fn new(is_connection_error: bool, error: String) -> Self {
+		if is_connection_error {
+			StringifiedMaybeConnectionError::Connection(error)
+		} else {
+			StringifiedMaybeConnectionError::NonConnection(error)
+		}
+	}
+}
+
+impl MaybeConnectionError for StringifiedMaybeConnectionError {
+	fn is_connection_error(&self) -> bool {
+		match *self {
+			StringifiedMaybeConnectionError::Connection(_) => true,
+			StringifiedMaybeConnectionError::NonConnection(_) => false,
+		}
+	}
+}
+
+impl ToString for StringifiedMaybeConnectionError {
+	fn to_string(&self) -> String {
+		match *self {
+			StringifiedMaybeConnectionError::Connection(ref err) => err.clone(),
+			StringifiedMaybeConnectionError::NonConnection(ref err) => err.clone(),
+		}
+	}
+}
+
+/// Exponential backoff for connection-unrelated errors retries.
+pub fn retry_backoff() -> ExponentialBackoff {
+	ExponentialBackoff {
+		// we do not want relayer to stop
+		max_elapsed_time: None,
+		max_interval: MAX_BACKOFF_INTERVAL,
+		..Default::default()
+	}
+}
+
+/// Compact format of IDs vector.
+pub fn format_ids<Id: std::fmt::Debug>(mut ids: impl ExactSizeIterator<Item = Id>) -> String {
+	const NTH_PROOF: &str = "we have checked len; qed";
+	match ids.len() {
+		0 => "<nothing>".into(),
+		1 => format!("{:?}", ids.next().expect(NTH_PROOF)),
+		2 => {
+			let id0 = ids.next().expect(NTH_PROOF);
+			let id1 = ids.next().expect(NTH_PROOF);
+			format!("[{:?}, {:?}]", id0, id1)
+		}
+		len => {
+			let id0 = ids.next().expect(NTH_PROOF);
+			let id_last = ids.last().expect(NTH_PROOF);
+			format!("{}:[{:?} ... {:?}]", len, id0, id_last)
+		}
+	}
+}
+
+/// Stream that emits item every `timeout_ms` milliseconds.
+pub fn interval(timeout: Duration) -> impl futures::Stream<Item = ()> {
+	futures::stream::unfold((), move |_| async move {
+		async_std::task::sleep(timeout).await;
+		Some(((), ()))
+	})
+}
+
+/// Which client has caused error.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum FailedClient {
+	/// It is the source client who has caused error.
+	Source,
+	/// It is the target client who has caused error.
+	Target,
+	/// Both clients are failing, or we just encountered some other error that
+	/// should be treated like that.
+	Both,
+}
+
+/// Future process result.
+#[derive(Debug, Clone, Copy)]
+pub enum ProcessFutureResult {
+	/// Future has been processed successfully.
+	Success,
+	/// Future has failed with non-connection error.
+	Failed,
+	/// Future has failed with connection error.
+	ConnectionFailed,
+}
+
+impl ProcessFutureResult {
+	/// Returns true if result is Success.
+	pub fn is_ok(self) -> bool {
+		match self {
+			ProcessFutureResult::Success => true,
+			ProcessFutureResult::Failed | ProcessFutureResult::ConnectionFailed => false,
+		}
+	}
+
+	/// Returns Ok(true) if future has succeeded.
+	/// Returns Ok(false) if future has failed with non-connection error.
+	/// Returns Err if future is `ConnectionFailed`.
+	pub fn fail_if_connection_error(self, failed_client: FailedClient) -> Result<bool, FailedClient> {
+		match self {
+			ProcessFutureResult::Success => Ok(true),
+			ProcessFutureResult::Failed => Ok(false),
+			ProcessFutureResult::ConnectionFailed => Err(failed_client),
+		}
+	}
+}
+
+/// Process result of the future from a client.
+pub fn process_future_result<TResult, TError, TGoOfflineFuture>(
+	result: Result<TResult, TError>,
+	retry_backoff: &mut ExponentialBackoff,
+	on_success: impl FnOnce(TResult),
+	go_offline_future: &mut std::pin::Pin<&mut futures::future::Fuse<TGoOfflineFuture>>,
+	go_offline: impl FnOnce(Duration) -> TGoOfflineFuture,
+	error_pattern: impl FnOnce() -> String,
+) -> ProcessFutureResult
+where
+	TError: std::fmt::Debug + MaybeConnectionError,
+	TGoOfflineFuture: FutureExt,
+{
+	match result {
+		Ok(result) => {
+			on_success(result);
+			retry_backoff.reset();
+			ProcessFutureResult::Success
+		}
+		Err(error) if error.is_connection_error() => {
+			log::error!(
+				target: "bridge",
+				"{}: {:?}. Going to restart",
+				error_pattern(),
+				error,
+			);
+
+			retry_backoff.reset();
+			go_offline_future.set(go_offline(CONNECTION_ERROR_DELAY).fuse());
+			ProcessFutureResult::ConnectionFailed
+		}
+		Err(error) => {
+			let retry_delay = retry_backoff.next_backoff().unwrap_or(CONNECTION_ERROR_DELAY);
+			log::error!(
+				target: "bridge",
+				"{}: {:?}. Retrying in {}",
+				error_pattern(),
+				error,
+				retry_delay.as_secs_f64(),
+			);
+
+			go_offline_future.set(go_offline(retry_delay).fuse());
+			ProcessFutureResult::Failed
+		}
+	}
+}

bridges/relays/generic/utils/src/metrics.rs

@@ -0,0 +1,168 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+pub use substrate_prometheus_endpoint::{register, Counter, CounterVec, Gauge, GaugeVec, Opts, Registry, F64, U64};
+
+use async_std::sync::{Arc, Mutex};
+use std::net::SocketAddr;
+use substrate_prometheus_endpoint::init_prometheus;
+use sysinfo::{ProcessExt, RefreshKind, System, SystemExt};
+
+/// Prometheus endpoint MetricsParams.
+#[derive(Debug, Clone)]
+pub struct MetricsParams {
+	/// Serve HTTP requests at given host.
+	pub host: String,
+	/// Serve HTTP requests at given port.
+	pub port: u16,
+}
+
+/// Metrics API.
+pub trait Metrics {
+	/// Register metrics in the registry.
+	fn register(&self, registry: &Registry) -> Result<(), String>;
+}
+
+/// Global Prometheus metrics.
+#[derive(Debug, Clone)]
+pub struct GlobalMetrics {
+	system: Arc<Mutex<System>>,
+	system_average_load: GaugeVec<F64>,
+	process_cpu_usage_percentage: Gauge<F64>,
+	process_memory_usage_bytes: Gauge<U64>,
+}
+
+/// Start Prometheus endpoint with given metrics registry.
+pub fn start(
+	prefix: String,
+	params: Option<MetricsParams>,
+	global_metrics: &GlobalMetrics,
+	extra_metrics: &impl Metrics,
+) {
+	let params = match params {
+		Some(params) => params,
+		None => return,
+	};
+
+	assert!(!prefix.is_empty(), "Metrics prefix can not be empty");
+
+	let do_start = move || {
+		let prometheus_socket_addr = SocketAddr::new(
+			params
+				.host
+				.parse()
+				.map_err(|err| format!("Invalid Prometheus host {}: {}", params.host, err))?,
+			params.port,
+		);
+		let metrics_registry =
+			Registry::new_custom(Some(prefix), None).expect("only fails if prefix is empty; prefix is not empty; qed");
+		global_metrics.register(&metrics_registry)?;
+		extra_metrics.register(&metrics_registry)?;
+		async_std::task::spawn(async move {
+			init_prometheus(prometheus_socket_addr, metrics_registry)
+				.await
+				.map_err(|err| format!("Error starting Prometheus endpoint: {}", err))
+		});
+
+		Ok(())
+	};
+
+	let result: Result<(), String> = do_start();
+	if let Err(err) = result {
+		log::warn!(
+			target: "bridge",
+			"Failed to expose metrics: {}",
+			err,
+		);
+	}
+}
+
+impl Default for MetricsParams {
+	fn default() -> Self {
+		MetricsParams {
+			host: "127.0.0.1".into(),
+			port: 9616,
+		}
+	}
+}
+
+impl Metrics for GlobalMetrics {
+	fn register(&self, registry: &Registry) -> Result<(), String> {
+		register(self.system_average_load.clone(), registry).map_err(|e| e.to_string())?;
+		register(self.process_cpu_usage_percentage.clone(), registry).map_err(|e| e.to_string())?;
+		register(self.process_memory_usage_bytes.clone(), registry).map_err(|e| e.to_string())?;
+		Ok(())
+	}
+}
+
+impl Default for GlobalMetrics {
+	fn default() -> Self {
+		GlobalMetrics {
+			system: Arc::new(Mutex::new(System::new_with_specifics(RefreshKind::everything()))),
+			system_average_load: GaugeVec::new(Opts::new("system_average_load", "System load average"), &["over"])
+				.expect("metric is static and thus valid; qed"),
+			process_cpu_usage_percentage: Gauge::new("process_cpu_usage_percentage", "Process CPU usage")
+				.expect("metric is static and thus valid; qed"),
+			process_memory_usage_bytes: Gauge::new(
+				"process_memory_usage_bytes",
+				"Process memory (resident set size) usage",
+			)
+			.expect("metric is static and thus valid; qed"),
+		}
+	}
+}
+
+impl GlobalMetrics {
+	/// Update metrics.
+	pub async fn update(&self) {
+		// update system-wide metrics
+		let mut system = self.system.lock().await;
+		let load = system.get_load_average();
+		self.system_average_load.with_label_values(&["1min"]).set(load.one);
+		self.system_average_load.with_label_values(&["5min"]).set(load.five);
+		self.system_average_load.with_label_values(&["15min"]).set(load.fifteen);
+
+		// update process-related metrics
+		let pid = sysinfo::get_current_pid().expect(
+			"only fails where pid is unavailable (os=unknown || arch=wasm32);\
+				relay is not supposed to run in such MetricsParamss;\
+				qed",
+		);
+		let is_process_refreshed = system.refresh_process(pid);
+		match (is_process_refreshed, system.get_process(pid)) {
+			(true, Some(process_info)) => {
+				let cpu_usage = process_info.cpu_usage() as f64;
+				let memory_usage = process_info.memory() * 1024;
+				log::trace!(
+					target: "bridge-metrics",
+					"Refreshed process metrics: CPU={}, memory={}",
+					cpu_usage,
+					memory_usage,
+				);
+
+				self.process_cpu_usage_percentage
+					.set(if cpu_usage.is_finite() { cpu_usage } else { 0f64 });
+				self.process_memory_usage_bytes.set(memory_usage);
+			}
+			_ => {
+				log::warn!(
+					target: "bridge",
+					"Failed to refresh process information. Metrics may show obsolete values",
+				);
+			}
+		}
+	}
+}

bridges/relays/generic/utils/src/relay_loop.rs

@@ -0,0 +1,95 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{FailedClient, MaybeConnectionError};
+
+use async_trait::async_trait;
+use std::{fmt::Debug, future::Future, time::Duration};
+
+/// Default pause between reconnect attempts.
+pub const RECONNECT_DELAY: Duration = Duration::from_secs(10);
+
+/// Basic blockchain client from relay perspective.
+#[async_trait]
+pub trait Client: Clone + Send + Sync {
+	/// Type of error this clients returns.
+	type Error: Debug + MaybeConnectionError;
+
+	/// Try to reconnect to source node.
+	async fn reconnect(&mut self) -> Result<(), Self::Error>;
+}
+
+/// Run relay loop.
+///
+/// This function represents an outer loop, which in turn calls provided `loop_run` function to do
+/// actual job. When `loop_run` returns, this outer loop reconnects to failed client (source,
+/// target or both) and calls `loop_run` again.
+pub fn run<SC: Client, TC: Client, R, F>(
+	reconnect_delay: Duration,
+	mut source_client: SC,
+	mut target_client: TC,
+	loop_run: R,
+) where
+	R: Fn(SC, TC) -> F,
+	F: Future<Output = Result<(), FailedClient>>,
+{
+	let mut local_pool = futures::executor::LocalPool::new();
+
+	local_pool.run_until(async move {
+		loop {
+			let result = loop_run(source_client.clone(), target_client.clone()).await;
+
+			match result {
+				Ok(()) => break,
+				Err(failed_client) => loop {
+					async_std::task::sleep(reconnect_delay).await;
+					if failed_client == FailedClient::Both || failed_client == FailedClient::Source {
+						match source_client.reconnect().await {
+							Ok(()) => (),
+							Err(error) => {
+								log::warn!(
+									target: "bridge",
+									"Failed to reconnect to source client. Going to retry in {}s: {:?}",
+									reconnect_delay.as_secs(),
+									error,
+								);
+								continue;
+							}
+						}
+					}
+					if failed_client == FailedClient::Both || failed_client == FailedClient::Target {
+						match target_client.reconnect().await {
+							Ok(()) => (),
+							Err(error) => {
+								log::warn!(
+									target: "bridge",
+									"Failed to reconnect to target client. Going to retry in {}s: {:?}",
+									reconnect_delay.as_secs(),
+									error,
+								);
+								continue;
+							}
+						}
+					}
+
+					break;
+				},
+			}
+
+			log::debug!(target: "bridge", "Restarting relay loop");
+		}
+	});
+}