Merge commit 'e5bed7ac380b6adb54b60a2a72a2a8f07f50d6c1' as 'bridges'

polkadot/bridges/relays/headers/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "headers-relay"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+
+[dependencies]
+async-std = "1.6.5"
+async-trait = "0.1.40"
+backoff = "0.2"
+futures = "0.3.5"
+linked-hash-map = "0.5.3"
+log = "0.4.11"
+num-traits = "0.2"
+parking_lot = "0.11.0"
+relay-utils = { path = "../utils" }

polkadot/bridges/relays/headers/src/lib.rs

@@ -0,0 +1,33 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Relaying source chain headers to target chain. This module provides entrypoint
+//! that starts reading new headers from source chain and submit these headers as
+//! module/contract transactions to the target chain. Pallet/contract on the target
+//! chain is a light-client of the source chain. All other trustless bridge
+//! applications are built using this light-client, so running headers-relay is
+//! essential for running all other bridge applications.
+
+// required for futures::select!
+#![recursion_limit = "1024"]
+#![warn(missing_docs)]
+
+pub mod headers;
+pub mod sync;
+pub mod sync_loop;
+pub mod sync_loop_metrics;
+pub mod sync_loop_tests;
+pub mod sync_types;

polkadot/bridges/relays/headers/src/sync.rs

@@ -0,0 +1,523 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Headers synchronization context. This structure wraps headers queue and is
+//! able to choose: which headers to read from the source chain? Which headers
+//! to submit to the target chain? The context makes decisions basing on parameters
+//! passed using `HeadersSyncParams` structure.
+
+use crate::headers::QueuedHeaders;
+use crate::sync_types::{HeaderIdOf, HeaderStatus, HeadersSyncPipeline, QueuedHeader};
+use num_traits::{One, Saturating, Zero};
+
+/// Common sync params.
+#[derive(Debug, Clone)]
+pub struct HeadersSyncParams {
+	/// Maximal number of ethereum headers to pre-download.
+	pub max_future_headers_to_download: usize,
+	/// Maximal number of active (we believe) submit header transactions.
+	pub max_headers_in_submitted_status: usize,
+	/// Maximal number of headers in single submit request.
+	pub max_headers_in_single_submit: usize,
+	/// Maximal total headers size in single submit request.
+	pub max_headers_size_in_single_submit: usize,
+	/// We only may store and accept (from Ethereum node) headers that have
+	/// number >= than best_substrate_header.number - prune_depth.
+	pub prune_depth: u32,
+	/// Target transactions mode.
+	pub target_tx_mode: TargetTransactionMode,
+}
+
+/// Target transaction mode.
+#[derive(Debug, PartialEq, Clone)]
+pub enum TargetTransactionMode {
+	/// Submit new headers using signed transactions.
+	Signed,
+	/// Submit new headers using unsigned transactions.
+	Unsigned,
+	/// Submit new headers using signed transactions, but only when we
+	/// believe that sync has stalled.
+	Backup,
+}
+
+/// Headers synchronization context.
+#[derive(Debug)]
+pub struct HeadersSync<P: HeadersSyncPipeline> {
+	/// Synchronization parameters.
+	params: HeadersSyncParams,
+	/// Best header number known to source node.
+	source_best_number: Option<P::Number>,
+	/// Best header known to target node.
+	target_best_header: Option<HeaderIdOf<P>>,
+	/// Headers queue.
+	headers: QueuedHeaders<P>,
+	/// Pause headers submission.
+	pause_submit: bool,
+}
+
+impl<P: HeadersSyncPipeline> HeadersSync<P> {
+	/// Creates new headers synchronizer.
+	pub fn new(params: HeadersSyncParams) -> Self {
+		HeadersSync {
+			headers: QueuedHeaders::default(),
+			params,
+			source_best_number: None,
+			target_best_header: None,
+			pause_submit: false,
+		}
+	}
+
+	/// Return best header number known to source node.
+	pub fn source_best_number(&self) -> Option<P::Number> {
+		self.source_best_number
+	}
+
+	/// Best header known to target node.
+	pub fn target_best_header(&self) -> Option<HeaderIdOf<P>> {
+		self.target_best_header
+	}
+
+	/// Returns true if we have synced almost all known headers.
+	pub fn is_almost_synced(&self) -> bool {
+		match self.source_best_number {
+			Some(source_best_number) => self
+				.target_best_header
+				.map(|best| source_best_number.saturating_sub(best.0) < 4.into())
+				.unwrap_or(false),
+			None => true,
+		}
+	}
+
+	/// Returns synchronization status.
+	pub fn status(&self) -> (&Option<HeaderIdOf<P>>, &Option<P::Number>) {
+		(&self.target_best_header, &self.source_best_number)
+	}
+
+	/// Returns reference to the headers queue.
+	pub fn headers(&self) -> &QueuedHeaders<P> {
+		&self.headers
+	}
+
+	/// Returns mutable reference to the headers queue.
+	pub fn headers_mut(&mut self) -> &mut QueuedHeaders<P> {
+		&mut self.headers
+	}
+
+	/// Select header that needs to be downloaded from the source node.
+	pub fn select_new_header_to_download(&self) -> Option<P::Number> {
+		// if we haven't received best header from source node yet, there's nothing we can download
+		let source_best_number = self.source_best_number?;
+
+		// if we haven't received known best header from target node yet, there's nothing we can download
+		let target_best_header = self.target_best_header.as_ref()?;
+
+		// if there's too many headers in the queue, stop downloading
+		let in_memory_headers = self.headers.total_headers();
+		if in_memory_headers >= self.params.max_future_headers_to_download {
+			return None;
+		}
+
+		// if queue is empty and best header on target is > than best header on source,
+		// then we shoud reorg
+		let best_queued_number = self.headers.best_queued_number();
+		if best_queued_number.is_zero() && source_best_number < target_best_header.0 {
+			return Some(source_best_number);
+		}
+
+		// we assume that there were no reorgs if we have already downloaded best header
+		let best_downloaded_number = std::cmp::max(
+			std::cmp::max(best_queued_number, self.headers.best_synced_number()),
+			target_best_header.0,
+		);
+		if best_downloaded_number >= source_best_number {
+			return None;
+		}
+
+		// download new header
+		Some(best_downloaded_number + One::one())
+	}
+
+	/// Selech orphan header to downoload.
+	pub fn select_orphan_header_to_download(&self) -> Option<&QueuedHeader<P>> {
+		let orphan_header = self.headers.header(HeaderStatus::Orphan)?;
+
+		// we consider header orphan until we'll find it ancestor that is known to the target node
+		// => we may get orphan header while we ask target node whether it knows its parent
+		// => let's avoid fetching duplicate headers
+		let parent_id = orphan_header.parent_id();
+		if self.headers.status(&parent_id) != HeaderStatus::Unknown {
+			return None;
+		}
+
+		Some(orphan_header)
+	}
+
+	/// Select headers that need to be submitted to the target node.
+	pub fn select_headers_to_submit(&self, stalled: bool) -> Option<Vec<&QueuedHeader<P>>> {
+		// maybe we have paused new headers submit?
+		if self.pause_submit {
+			return None;
+		}
+
+		// if we operate in backup mode, we only submit headers when sync has stalled
+		if self.params.target_tx_mode == TargetTransactionMode::Backup && !stalled {
+			return None;
+		}
+
+		let headers_in_submit_status = self.headers.headers_in_status(HeaderStatus::Submitted);
+		let headers_to_submit_count = self
+			.params
+			.max_headers_in_submitted_status
+			.checked_sub(headers_in_submit_status)?;
+
+		let mut total_size = 0;
+		let mut total_headers = 0;
+		self.headers.headers(HeaderStatus::Ready, |header| {
+			if total_headers == headers_to_submit_count {
+				return false;
+			}
+			if total_headers == self.params.max_headers_in_single_submit {
+				return false;
+			}
+
+			let encoded_size = P::estimate_size(header);
+			if total_headers != 0 && total_size + encoded_size > self.params.max_headers_size_in_single_submit {
+				return false;
+			}
+
+			total_size += encoded_size;
+			total_headers += 1;
+
+			true
+		})
+	}
+
+	/// Receive new target header number from the source node.
+	pub fn source_best_header_number_response(&mut self, best_header_number: P::Number) {
+		log::debug!(
+			target: "bridge",
+			"Received best header number from {} node: {}",
+			P::SOURCE_NAME,
+			best_header_number,
+		);
+		self.source_best_number = Some(best_header_number);
+	}
+
+	/// Receive new best header from the target node.
+	/// Returns true if it is different from the previous block known to us.
+	pub fn target_best_header_response(&mut self, best_header: HeaderIdOf<P>) -> bool {
+		log::debug!(
+			target: "bridge",
+			"Received best known header from {}: {:?}",
+			P::TARGET_NAME,
+			best_header,
+		);
+
+		// early return if it is still the same
+		if self.target_best_header == Some(best_header) {
+			return false;
+		}
+
+		// remember that this header is now known to the Substrate runtime
+		self.headers.target_best_header_response(&best_header);
+
+		// prune ancient headers
+		self.headers
+			.prune(best_header.0.saturating_sub(self.params.prune_depth.into()));
+
+		// finally remember the best header itself
+		self.target_best_header = Some(best_header);
+
+		// we are ready to submit headers again
+		if self.pause_submit {
+			log::debug!(
+				target: "bridge",
+				"Ready to submit {} headers to {} node again!",
+				P::SOURCE_NAME,
+				P::TARGET_NAME,
+			);
+
+			self.pause_submit = false;
+		}
+
+		true
+	}
+
+	/// Pause headers submit until best header will be updated on target node.
+	pub fn pause_submit(&mut self) {
+		log::debug!(
+			target: "bridge",
+			"Stopping submitting {} headers to {} node. Waiting for {} submitted headers to be accepted",
+			P::SOURCE_NAME,
+			P::TARGET_NAME,
+			self.headers.headers_in_status(HeaderStatus::Submitted),
+		);
+
+		self.pause_submit = true;
+	}
+
+	/// Restart synchronization.
+	pub fn restart(&mut self) {
+		self.source_best_number = None;
+		self.target_best_header = None;
+		self.headers.clear();
+		self.pause_submit = false;
+	}
+}
+
+#[cfg(test)]
+pub mod tests {
+	use super::*;
+	use crate::headers::tests::{header, id};
+	use crate::sync_loop_tests::{TestHash, TestHeadersSyncPipeline, TestNumber};
+	use crate::sync_types::HeaderStatus;
+	use relay_utils::HeaderId;
+
+	fn side_hash(number: TestNumber) -> TestHash {
+		1000 + number
+	}
+
+	pub fn default_sync_params() -> HeadersSyncParams {
+		HeadersSyncParams {
+			max_future_headers_to_download: 128,
+			max_headers_in_submitted_status: 128,
+			max_headers_in_single_submit: 32,
+			max_headers_size_in_single_submit: 131_072,
+			prune_depth: 4096,
+			target_tx_mode: TargetTransactionMode::Signed,
+		}
+	}
+
+	#[test]
+	fn select_new_header_to_download_works() {
+		let mut eth_sync = HeadersSync::<TestHeadersSyncPipeline>::new(default_sync_params());
+
+		// both best && target headers are unknown
+		assert_eq!(eth_sync.select_new_header_to_download(), None);
+
+		// best header is known, target header is unknown
+		eth_sync.target_best_header = Some(HeaderId(0, Default::default()));
+		assert_eq!(eth_sync.select_new_header_to_download(), None);
+
+		// target header is known, best header is unknown
+		eth_sync.target_best_header = None;
+		eth_sync.source_best_number = Some(100);
+		assert_eq!(eth_sync.select_new_header_to_download(), None);
+
+		// when our best block has the same number as the target
+		eth_sync.target_best_header = Some(HeaderId(100, Default::default()));
+		assert_eq!(eth_sync.select_new_header_to_download(), None);
+
+		// when we actually need a new header
+		eth_sync.source_best_number = Some(101);
+		assert_eq!(eth_sync.select_new_header_to_download(), Some(101));
+
+		// when we have to reorganize to longer fork
+		eth_sync.source_best_number = Some(100);
+		eth_sync.target_best_header = Some(HeaderId(200, Default::default()));
+		assert_eq!(eth_sync.select_new_header_to_download(), Some(100));
+
+		// when there are too many headers scheduled for submitting
+		for i in 1..1000 {
+			eth_sync.headers.header_response(header(i).header().clone());
+		}
+		assert_eq!(eth_sync.select_new_header_to_download(), None);
+	}
+
+	#[test]
+	fn select_new_header_to_download_works_with_empty_queue() {
+		let mut eth_sync = HeadersSync::<TestHeadersSyncPipeline>::new(default_sync_params());
+		eth_sync.source_best_header_number_response(100);
+
+		// when queue is not empty => everything goes as usually
+		eth_sync.target_best_header_response(header(10).id());
+		eth_sync.headers_mut().header_response(header(11).header().clone());
+		eth_sync.headers_mut().maybe_extra_response(&header(11).id(), false);
+		assert_eq!(eth_sync.select_new_header_to_download(), Some(12));
+
+		// but then queue is drained
+		eth_sync.headers_mut().target_best_header_response(&header(11).id());
+
+		// even though it's empty, we know that header#11 is synced
+		assert_eq!(eth_sync.headers().best_queued_number(), 0);
+		assert_eq!(eth_sync.headers().best_synced_number(), 11);
+		assert_eq!(eth_sync.select_new_header_to_download(), Some(12));
+	}
+
+	#[test]
+	fn sync_without_reorgs_works() {
+		let mut eth_sync = HeadersSync::new(default_sync_params());
+		eth_sync.params.max_headers_in_submitted_status = 1;
+
+		// ethereum reports best header #102
+		eth_sync.source_best_header_number_response(102);
+
+		// substrate reports that it is at block #100
+		eth_sync.target_best_header_response(id(100));
+
+		// block #101 is downloaded first
+		assert_eq!(eth_sync.select_new_header_to_download(), Some(101));
+		eth_sync.headers.header_response(header(101).header().clone());
+
+		// now header #101 is ready to be submitted
+		assert_eq!(eth_sync.headers.header(HeaderStatus::MaybeExtra), Some(&header(101)));
+		eth_sync.headers.maybe_extra_response(&id(101), false);
+		assert_eq!(eth_sync.headers.header(HeaderStatus::Ready), Some(&header(101)));
+		assert_eq!(eth_sync.select_headers_to_submit(false), Some(vec![&header(101)]));
+
+		// and header #102 is ready to be downloaded
+		assert_eq!(eth_sync.select_new_header_to_download(), Some(102));
+		eth_sync.headers.header_response(header(102).header().clone());
+
+		// receive submission confirmation
+		eth_sync.headers.headers_submitted(vec![id(101)]);
+
+		// we have nothing to submit because previous header hasn't been confirmed yet
+		// (and we allow max 1 submit transaction in the wild)
+		assert_eq!(eth_sync.headers.header(HeaderStatus::MaybeExtra), Some(&header(102)));
+		eth_sync.headers.maybe_extra_response(&id(102), false);
+		assert_eq!(eth_sync.headers.header(HeaderStatus::Ready), Some(&header(102)));
+		assert_eq!(eth_sync.select_headers_to_submit(false), None);
+
+		// substrate reports that it has imported block #101
+		eth_sync.target_best_header_response(id(101));
+
+		// and we are ready to submit #102
+		assert_eq!(eth_sync.select_headers_to_submit(false), Some(vec![&header(102)]));
+		eth_sync.headers.headers_submitted(vec![id(102)]);
+
+		// substrate reports that it has imported block #102
+		eth_sync.target_best_header_response(id(102));
+
+		// and we have nothing to download
+		assert_eq!(eth_sync.select_new_header_to_download(), None);
+	}
+
+	#[test]
+	fn sync_with_orphan_headers_work() {
+		let mut eth_sync = HeadersSync::new(default_sync_params());
+
+		// ethereum reports best header #102
+		eth_sync.source_best_header_number_response(102);
+
+		// substrate reports that it is at block #100, but it isn't part of best chain
+		eth_sync.target_best_header_response(HeaderId(100, side_hash(100)));
+
+		// block #101 is downloaded first
+		assert_eq!(eth_sync.select_new_header_to_download(), Some(101));
+		eth_sync.headers.header_response(header(101).header().clone());
+
+		// we can't submit header #101, because its parent status is unknown
+		assert_eq!(eth_sync.select_headers_to_submit(false), None);
+
+		// instead we are trying to determine status of its parent (#100)
+		assert_eq!(eth_sync.headers.header(HeaderStatus::MaybeOrphan), Some(&header(101)));
+
+		// and the status is still unknown
+		eth_sync.headers.maybe_orphan_response(&id(100), false);
+
+		// so we consider #101 orphaned now && will download its parent - #100
+		assert_eq!(eth_sync.headers.header(HeaderStatus::Orphan), Some(&header(101)));
+		eth_sync.headers.header_response(header(100).header().clone());
+
+		// #101 is now Orphan and #100 is MaybeOrphan => we do not want to retrieve
+		// header #100 again
+		assert_eq!(eth_sync.headers.header(HeaderStatus::Orphan), Some(&header(101)));
+		assert_eq!(eth_sync.select_orphan_header_to_download(), None);
+
+		// we can't submit header #100, because its parent status is unknown
+		assert_eq!(eth_sync.select_headers_to_submit(false), None);
+
+		// instead we are trying to determine status of its parent (#99)
+		assert_eq!(eth_sync.headers.header(HeaderStatus::MaybeOrphan), Some(&header(100)));
+
+		// and the status is known, so we move previously orphaned #100 and #101 to ready queue
+		eth_sync.headers.maybe_orphan_response(&id(99), true);
+
+		// and we are ready to submit #100
+		assert_eq!(eth_sync.headers.header(HeaderStatus::MaybeExtra), Some(&header(100)));
+		eth_sync.headers.maybe_extra_response(&id(100), false);
+		assert_eq!(eth_sync.select_headers_to_submit(false), Some(vec![&header(100)]));
+		eth_sync.headers.headers_submitted(vec![id(100)]);
+
+		// and we are ready to submit #101
+		assert_eq!(eth_sync.headers.header(HeaderStatus::MaybeExtra), Some(&header(101)));
+		eth_sync.headers.maybe_extra_response(&id(101), false);
+		assert_eq!(eth_sync.select_headers_to_submit(false), Some(vec![&header(101)]));
+		eth_sync.headers.headers_submitted(vec![id(101)]);
+	}
+
+	#[test]
+	fn pruning_happens_on_target_best_header_response() {
+		let mut eth_sync = HeadersSync::<TestHeadersSyncPipeline>::new(default_sync_params());
+		eth_sync.params.prune_depth = 50;
+		eth_sync.target_best_header_response(id(100));
+		assert_eq!(eth_sync.headers.prune_border(), 50);
+	}
+
+	#[test]
+	fn only_submitting_headers_in_backup_mode_when_stalled() {
+		let mut eth_sync = HeadersSync::new(default_sync_params());
+		eth_sync.params.target_tx_mode = TargetTransactionMode::Backup;
+
+		// ethereum reports best header #102
+		eth_sync.source_best_header_number_response(102);
+
+		// substrate reports that it is at block #100
+		eth_sync.target_best_header_response(id(100));
+
+		// block #101 is downloaded first
+		eth_sync.headers.header_response(header(101).header().clone());
+		eth_sync.headers.maybe_extra_response(&id(101), false);
+
+		// ensure that headers are not submitted when sync is not stalled
+		assert_eq!(eth_sync.select_headers_to_submit(false), None);
+
+		// ensure that headers are not submitted when sync is stalled
+		assert_eq!(eth_sync.select_headers_to_submit(true), Some(vec![&header(101)]));
+	}
+
+	#[test]
+	fn does_not_select_new_headers_to_submit_when_submit_is_paused() {
+		let mut eth_sync = HeadersSync::new(default_sync_params());
+		eth_sync.params.max_headers_in_submitted_status = 1;
+
+		// ethereum reports best header #102 and substrate is at #100
+		eth_sync.source_best_header_number_response(102);
+		eth_sync.target_best_header_response(id(100));
+
+		// let's prepare #101 and #102 for submitting
+		eth_sync.headers.header_response(header(101).header().clone());
+		eth_sync.headers.maybe_extra_response(&id(101), false);
+		eth_sync.headers.header_response(header(102).header().clone());
+		eth_sync.headers.maybe_extra_response(&id(102), false);
+
+		// when submit is not paused, we're ready to submit #101
+		assert_eq!(eth_sync.select_headers_to_submit(false), Some(vec![&header(101)]));
+
+		// when submit is paused, we're not ready to submit anything
+		eth_sync.pause_submit();
+		assert_eq!(eth_sync.select_headers_to_submit(false), None);
+
+		// if best header on substrate node isn't updated, we still not submitting anything
+		eth_sync.target_best_header_response(id(100));
+		assert_eq!(eth_sync.select_headers_to_submit(false), None);
+
+		// but after it is actually updated, we are ready to submit
+		eth_sync.target_best_header_response(id(101));
+		assert_eq!(eth_sync.select_headers_to_submit(false), Some(vec![&header(102)]));
+	}
+}

polkadot/bridges/relays/headers/src/sync_loop.rs

@@ -0,0 +1,637 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Entrypoint for running headers synchronization loop.
+
+use crate::sync::{HeadersSync, HeadersSyncParams};
+use crate::sync_loop_metrics::SyncLoopMetrics;
+use crate::sync_types::{HeaderIdOf, HeaderStatus, HeadersSyncPipeline, QueuedHeader, SubmittedHeaders};
+
+use async_trait::async_trait;
+use futures::{future::FutureExt, stream::StreamExt};
+use num_traits::{Saturating, Zero};
+use relay_utils::{
+	format_ids, interval,
+	metrics::{GlobalMetrics, MetricsParams},
+	process_future_result,
+	relay_loop::Client as RelayClient,
+	retry_backoff, FailedClient, MaybeConnectionError, StringifiedMaybeConnectionError,
+};
+use std::{
+	collections::HashSet,
+	future::Future,
+	time::{Duration, Instant},
+};
+
+/// When we submit headers to target node, but see no updates of best
+/// source block known to target node during STALL_SYNC_TIMEOUT seconds,
+/// we consider that our headers are rejected because there has been reorg in target chain.
+/// This reorg could invalidate our knowledge about sync process (i.e. we have asked if
+/// HeaderA is known to target, but then reorg happened and the answer is different
+/// now) => we need to reset sync.
+/// The other option is to receive **EVERY** best target header and check if it is
+/// direct child of previous best header. But: (1) subscription doesn't guarantee that
+/// the subscriber will receive every best header (2) reorg won't always lead to sync
+/// stall and restart is a heavy operation (we forget all in-memory headers).
+const STALL_SYNC_TIMEOUT: Duration = Duration::from_secs(5 * 60);
+/// Delay after we have seen update of best source header at target node,
+/// for us to treat sync stalled. ONLY when relay operates in backup mode.
+const BACKUP_STALL_SYNC_TIMEOUT: Duration = Duration::from_secs(10 * 60);
+/// Interval between calling sync maintain procedure.
+const MAINTAIN_INTERVAL: Duration = Duration::from_secs(30);
+
+/// Source client trait.
+#[async_trait]
+pub trait SourceClient<P: HeadersSyncPipeline>: RelayClient {
+	/// Get best block number.
+	async fn best_block_number(&self) -> Result<P::Number, Self::Error>;
+
+	/// Get header by hash.
+	async fn header_by_hash(&self, hash: P::Hash) -> Result<P::Header, Self::Error>;
+
+	/// Get canonical header by number.
+	async fn header_by_number(&self, number: P::Number) -> Result<P::Header, Self::Error>;
+
+	/// Get completion data by header hash.
+	async fn header_completion(&self, id: HeaderIdOf<P>)
+		-> Result<(HeaderIdOf<P>, Option<P::Completion>), Self::Error>;
+
+	/// Get extra data by header hash.
+	async fn header_extra(
+		&self,
+		id: HeaderIdOf<P>,
+		header: QueuedHeader<P>,
+	) -> Result<(HeaderIdOf<P>, P::Extra), Self::Error>;
+}
+
+/// Target client trait.
+#[async_trait]
+pub trait TargetClient<P: HeadersSyncPipeline>: RelayClient {
+	/// Returns ID of best header known to the target node.
+	async fn best_header_id(&self) -> Result<HeaderIdOf<P>, Self::Error>;
+
+	/// Returns true if header is known to the target node.
+	async fn is_known_header(&self, id: HeaderIdOf<P>) -> Result<(HeaderIdOf<P>, bool), Self::Error>;
+
+	/// Submit headers.
+	async fn submit_headers(&self, headers: Vec<QueuedHeader<P>>) -> SubmittedHeaders<HeaderIdOf<P>, Self::Error>;
+
+	/// Returns ID of headers that require to be 'completed' before children can be submitted.
+	async fn incomplete_headers_ids(&self) -> Result<HashSet<HeaderIdOf<P>>, Self::Error>;
+
+	/// Submit completion data for header.
+	async fn complete_header(&self, id: HeaderIdOf<P>, completion: P::Completion)
+		-> Result<HeaderIdOf<P>, Self::Error>;
+
+	/// Returns true if header requires extra data to be submitted.
+	async fn requires_extra(&self, header: QueuedHeader<P>) -> Result<(HeaderIdOf<P>, bool), Self::Error>;
+}
+
+/// Synchronization maintain procedure.
+#[async_trait]
+pub trait SyncMaintain<P: HeadersSyncPipeline>: Clone + Send + Sync {
+	/// Run custom maintain procedures. This is guaranteed to be called when both source and target
+	/// clients are unoccupied.
+	async fn maintain(&self, _sync: &mut HeadersSync<P>) {}
+}
+
+impl<P: HeadersSyncPipeline> SyncMaintain<P> for () {}
+
+/// Return prefix that will be used by default to expose Prometheus metrics of the finality proofs sync loop.
+pub fn metrics_prefix<P: HeadersSyncPipeline>() -> String {
+	format!("{}_to_{}_Sync", P::SOURCE_NAME, P::TARGET_NAME)
+}
+
+/// Run headers synchronization.
+#[allow(clippy::too_many_arguments)]
+pub async fn run<P: HeadersSyncPipeline, TC: TargetClient<P>>(
+	source_client: impl SourceClient<P>,
+	source_tick: Duration,
+	target_client: TC,
+	target_tick: Duration,
+	sync_maintain: impl SyncMaintain<P>,
+	sync_params: HeadersSyncParams,
+	metrics_params: MetricsParams,
+	exit_signal: impl Future<Output = ()>,
+) -> Result<(), String> {
+	let exit_signal = exit_signal.shared();
+	relay_utils::relay_loop(source_client, target_client)
+		.with_metrics(Some(metrics_prefix::<P>()), metrics_params)
+		.loop_metric(|registry, prefix| SyncLoopMetrics::new(registry, prefix))?
+		.standalone_metric(|registry, prefix| GlobalMetrics::new(registry, prefix))?
+		.expose()
+		.await?
+		.run(|source_client, target_client, metrics| {
+			run_until_connection_lost(
+				source_client,
+				source_tick,
+				target_client,
+				target_tick,
+				sync_maintain.clone(),
+				sync_params.clone(),
+				metrics,
+				exit_signal.clone(),
+			)
+		})
+		.await
+}
+
+/// Run headers synchronization.
+#[allow(clippy::too_many_arguments)]
+async fn run_until_connection_lost<P: HeadersSyncPipeline, TC: TargetClient<P>>(
+	source_client: impl SourceClient<P>,
+	source_tick: Duration,
+	target_client: TC,
+	target_tick: Duration,
+	sync_maintain: impl SyncMaintain<P>,
+	sync_params: HeadersSyncParams,
+	metrics_sync: Option<SyncLoopMetrics>,
+	exit_signal: impl Future<Output = ()>,
+) -> Result<(), FailedClient> {
+	let mut progress_context = (Instant::now(), None, None);
+
+	let mut sync = HeadersSync::<P>::new(sync_params);
+	let mut stall_countdown = None;
+	let mut last_update_time = Instant::now();
+
+	let mut source_retry_backoff = retry_backoff();
+	let mut source_client_is_online = false;
+	let mut source_best_block_number_required = false;
+	let source_best_block_number_future = source_client.best_block_number().fuse();
+	let source_new_header_future = futures::future::Fuse::terminated();
+	let source_orphan_header_future = futures::future::Fuse::terminated();
+	let source_extra_future = futures::future::Fuse::terminated();
+	let source_completion_future = futures::future::Fuse::terminated();
+	let source_go_offline_future = futures::future::Fuse::terminated();
+	let source_tick_stream = interval(source_tick).fuse();
+
+	let mut target_retry_backoff = retry_backoff();
+	let mut target_client_is_online = false;
+	let mut target_best_block_required = false;
+	let mut target_incomplete_headers_required = true;
+	let target_best_block_future = target_client.best_header_id().fuse();
+	let target_incomplete_headers_future = futures::future::Fuse::terminated();
+	let target_extra_check_future = futures::future::Fuse::terminated();
+	let target_existence_status_future = futures::future::Fuse::terminated();
+	let target_submit_header_future = futures::future::Fuse::terminated();
+	let target_complete_header_future = futures::future::Fuse::terminated();
+	let target_go_offline_future = futures::future::Fuse::terminated();
+	let target_tick_stream = interval(target_tick).fuse();
+
+	let mut maintain_required = false;
+	let maintain_stream = interval(MAINTAIN_INTERVAL).fuse();
+
+	let exit_signal = exit_signal.fuse();
+
+	futures::pin_mut!(
+		source_best_block_number_future,
+		source_new_header_future,
+		source_orphan_header_future,
+		source_extra_future,
+		source_completion_future,
+		source_go_offline_future,
+		source_tick_stream,
+		target_best_block_future,
+		target_incomplete_headers_future,
+		target_extra_check_future,
+		target_existence_status_future,
+		target_submit_header_future,
+		target_complete_header_future,
+		target_go_offline_future,
+		target_tick_stream,
+		maintain_stream,
+		exit_signal
+	);
+
+	loop {
+		futures::select! {
+			source_best_block_number = source_best_block_number_future => {
+				source_best_block_number_required = false;
+
+				source_client_is_online = process_future_result(
+					source_best_block_number,
+					&mut source_retry_backoff,
+					|source_best_block_number| sync.source_best_header_number_response(source_best_block_number),
+					&mut source_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving best header number from {}", P::SOURCE_NAME),
+				).fail_if_connection_error(FailedClient::Source)?;
+			},
+			source_new_header = source_new_header_future => {
+				source_client_is_online = process_future_result(
+					source_new_header,
+					&mut source_retry_backoff,
+					|source_new_header| sync.headers_mut().header_response(source_new_header),
+					&mut source_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving header from {} node", P::SOURCE_NAME),
+				).fail_if_connection_error(FailedClient::Source)?;
+			},
+			source_orphan_header = source_orphan_header_future => {
+				source_client_is_online = process_future_result(
+					source_orphan_header,
+					&mut source_retry_backoff,
+					|source_orphan_header| sync.headers_mut().header_response(source_orphan_header),
+					&mut source_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving orphan header from {} node", P::SOURCE_NAME),
+				).fail_if_connection_error(FailedClient::Source)?;
+			},
+			source_extra = source_extra_future => {
+				source_client_is_online = process_future_result(
+					source_extra,
+					&mut source_retry_backoff,
+					|(header, extra)| sync.headers_mut().extra_response(&header, extra),
+					&mut source_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving extra data from {} node", P::SOURCE_NAME),
+				).fail_if_connection_error(FailedClient::Source)?;
+			},
+			source_completion = source_completion_future => {
+				source_client_is_online = process_future_result(
+					source_completion,
+					&mut source_retry_backoff,
+					|(header, completion)| sync.headers_mut().completion_response(&header, completion),
+					&mut source_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving completion data from {} node", P::SOURCE_NAME),
+				).fail_if_connection_error(FailedClient::Source)?;
+			},
+			_ = source_go_offline_future => {
+				source_client_is_online = true;
+			},
+			_ = source_tick_stream.next() => {
+				if sync.is_almost_synced() {
+					source_best_block_number_required = true;
+				}
+			},
+			target_best_block = target_best_block_future => {
+				target_best_block_required = false;
+
+				target_client_is_online = process_future_result(
+					target_best_block,
+					&mut target_retry_backoff,
+					|target_best_block| {
+						let head_updated = sync.target_best_header_response(target_best_block);
+						if head_updated {
+							last_update_time = Instant::now();
+						}
+						match head_updated {
+							// IF head is updated AND there are still our transactions:
+							// => restart stall countdown timer
+							true if sync.headers().headers_in_status(HeaderStatus::Submitted) != 0 =>
+								stall_countdown = Some(Instant::now()),
+							// IF head is updated AND there are no our transactions:
+							// => stop stall countdown timer
+							true => stall_countdown = None,
+							// IF head is not updated AND stall countdown is not yet completed
+							// => do nothing
+							false if stall_countdown
+								.map(|stall_countdown| stall_countdown.elapsed() < STALL_SYNC_TIMEOUT)
+								.unwrap_or(true)
+								=> (),
+							// IF head is not updated AND stall countdown has completed
+							// => restart sync
+							false => {
+								log::info!(
+									target: "bridge",
+									"Sync has stalled. Restarting {} headers synchronization.",
+									P::SOURCE_NAME,
+								);
+								stall_countdown = None;
+								sync.restart();
+							},
+						}
+					},
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving best known {} header from {} node", P::SOURCE_NAME, P::TARGET_NAME),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+			incomplete_headers_ids = target_incomplete_headers_future => {
+				target_incomplete_headers_required = false;
+
+				target_client_is_online = process_future_result(
+					incomplete_headers_ids,
+					&mut target_retry_backoff,
+					|incomplete_headers_ids| sync.headers_mut().incomplete_headers_response(incomplete_headers_ids),
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving incomplete headers from {} node", P::TARGET_NAME),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+			target_existence_status = target_existence_status_future => {
+				target_client_is_online = process_future_result(
+					target_existence_status,
+					&mut target_retry_backoff,
+					|(target_header, target_existence_status)| sync
+						.headers_mut()
+						.maybe_orphan_response(&target_header, target_existence_status),
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving existence status from {} node", P::TARGET_NAME),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+			submitted_headers = target_submit_header_future => {
+				// following line helps Rust understand the type of `submitted_headers` :/
+				let submitted_headers: SubmittedHeaders<HeaderIdOf<P>, TC::Error> = submitted_headers;
+				let submitted_headers_str = format!("{}", submitted_headers);
+				let all_headers_rejected = submitted_headers.submitted.is_empty()
+					&& submitted_headers.incomplete.is_empty();
+				let has_submitted_headers = sync.headers().headers_in_status(HeaderStatus::Submitted) != 0;
+
+				let maybe_fatal_error = match submitted_headers.fatal_error {
+					Some(fatal_error) => Err(StringifiedMaybeConnectionError::new(
+						fatal_error.is_connection_error(),
+						format!("{:?}", fatal_error),
+					)),
+					None if all_headers_rejected && !has_submitted_headers =>
+						Err(StringifiedMaybeConnectionError::new(false, "All headers were rejected".into())),
+					None => Ok(()),
+				};
+
+				let no_fatal_error = maybe_fatal_error.is_ok();
+				target_client_is_online = process_future_result(
+					maybe_fatal_error,
+					&mut target_retry_backoff,
+					|_| {},
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error submitting headers to {} node", P::TARGET_NAME),
+				).fail_if_connection_error(FailedClient::Target)?;
+
+				log::debug!(target: "bridge", "Header submit result: {}", submitted_headers_str);
+
+				sync.headers_mut().headers_submitted(submitted_headers.submitted);
+				sync.headers_mut().add_incomplete_headers(false, submitted_headers.incomplete);
+
+				// when there's no fatal error, but node has rejected all our headers we may
+				// want to pause until our submitted headers will be accepted
+				if no_fatal_error && all_headers_rejected && has_submitted_headers {
+					sync.pause_submit();
+				}
+			},
+			target_complete_header_result = target_complete_header_future => {
+				target_client_is_online = process_future_result(
+					target_complete_header_result,
+					&mut target_retry_backoff,
+					|completed_header| sync.headers_mut().header_completed(&completed_header),
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error completing headers at {}", P::TARGET_NAME),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+			target_extra_check_result = target_extra_check_future => {
+				target_client_is_online = process_future_result(
+					target_extra_check_result,
+					&mut target_retry_backoff,
+					|(header, extra_check_result)| sync
+						.headers_mut()
+						.maybe_extra_response(&header, extra_check_result),
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving receipts requirement from {} node", P::TARGET_NAME),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+			_ = target_go_offline_future => {
+				target_client_is_online = true;
+			},
+			_ = target_tick_stream.next() => {
+				target_best_block_required = true;
+				target_incomplete_headers_required = true;
+			},
+
+			_ = maintain_stream.next() => {
+				maintain_required = true;
+			},
+			_ = exit_signal => {
+				return Ok(());
+			}
+		}
+
+		// update metrics
+		if let Some(ref metrics_sync) = metrics_sync {
+			metrics_sync.update(&sync);
+		}
+
+		// print progress
+		progress_context = print_sync_progress(progress_context, &sync);
+
+		// run maintain procedures
+		if maintain_required && source_client_is_online && target_client_is_online {
+			log::debug!(target: "bridge", "Maintaining headers sync loop");
+			maintain_required = false;
+			sync_maintain.maintain(&mut sync).await;
+		}
+
+		// If the target client is accepting requests we update the requests that
+		// we want it to run
+		if !maintain_required && target_client_is_online {
+			// NOTE: Is is important to reset this so that we only have one
+			// request being processed by the client at a time. This prevents
+			// race conditions like receiving two transactions with the same
+			// nonce from the client.
+			target_client_is_online = false;
+
+			// The following is how we prioritize requests:
+			//
+			// 1. Get best block
+			//     - Stops us from downloading or submitting new blocks
+			//     - Only called rarely
+			//
+			// 2. Get incomplete headers
+			//     - Stops us from submitting new blocks
+			//     - Only called rarely
+			//
+			// 3. Get complete headers
+			//     - Stops us from submitting new blocks
+			//
+			// 4. Check if we need extra data from source
+			//     - Stops us from downloading or submitting new blocks
+			//
+			// 5. Check existence of header
+			//     - Stops us from submitting new blocks
+			//
+			// 6. Submit header
+
+			if target_best_block_required {
+				log::debug!(target: "bridge", "Asking {} about best block", P::TARGET_NAME);
+				target_best_block_future.set(target_client.best_header_id().fuse());
+			} else if target_incomplete_headers_required {
+				log::debug!(target: "bridge", "Asking {} about incomplete headers", P::TARGET_NAME);
+				target_incomplete_headers_future.set(target_client.incomplete_headers_ids().fuse());
+			} else if let Some((id, completion)) = sync.headers_mut().header_to_complete() {
+				log::debug!(
+					target: "bridge",
+					"Going to complete header: {:?}",
+					id,
+				);
+
+				target_complete_header_future.set(target_client.complete_header(id, completion.clone()).fuse());
+			} else if let Some(header) = sync.headers().header(HeaderStatus::MaybeExtra) {
+				log::debug!(
+					target: "bridge",
+					"Checking if header submission requires extra: {:?}",
+					header.id(),
+				);
+
+				target_extra_check_future.set(target_client.requires_extra(header.clone()).fuse());
+			} else if let Some(header) = sync.headers().header(HeaderStatus::MaybeOrphan) {
+				// for MaybeOrphan we actually ask for parent' header existence
+				let parent_id = header.parent_id();
+
+				log::debug!(
+					target: "bridge",
+					"Asking {} node for existence of: {:?}",
+					P::TARGET_NAME,
+					parent_id,
+				);
+
+				target_existence_status_future.set(target_client.is_known_header(parent_id).fuse());
+			} else if let Some(headers) =
+				sync.select_headers_to_submit(last_update_time.elapsed() > BACKUP_STALL_SYNC_TIMEOUT)
+			{
+				log::debug!(
+					target: "bridge",
+					"Submitting {} header(s) to {} node: {:?}",
+					headers.len(),
+					P::TARGET_NAME,
+					format_ids(headers.iter().map(|header| header.id())),
+				);
+
+				let headers = headers.into_iter().cloned().collect();
+				target_submit_header_future.set(target_client.submit_headers(headers).fuse());
+
+				// remember that we have submitted some headers
+				if stall_countdown.is_none() {
+					stall_countdown = Some(Instant::now());
+				}
+			} else {
+				target_client_is_online = true;
+			}
+		}
+
+		// If the source client is accepting requests we update the requests that
+		// we want it to run
+		if !maintain_required && source_client_is_online {
+			// NOTE: Is is important to reset this so that we only have one
+			// request being processed by the client at a time. This prevents
+			// race conditions like receiving two transactions with the same
+			// nonce from the client.
+			source_client_is_online = false;
+
+			// The following is how we prioritize requests:
+			//
+			// 1. Get best block
+			//     - Stops us from downloading or submitting new blocks
+			//     - Only called rarely
+			//
+			// 2. Download completion data
+			//     - Stops us from submitting new blocks
+			//
+			// 3. Download extra data
+			//     - Stops us from submitting new blocks
+			//
+			// 4. Download missing headers
+			//     - Stops us from downloading or submitting new blocks
+			//
+			// 5. Downloading new headers
+
+			if source_best_block_number_required {
+				log::debug!(target: "bridge", "Asking {} node about best block number", P::SOURCE_NAME);
+				source_best_block_number_future.set(source_client.best_block_number().fuse());
+			} else if let Some(id) = sync.headers_mut().incomplete_header() {
+				log::debug!(
+					target: "bridge",
+					"Retrieving completion data for header: {:?}",
+					id,
+				);
+				source_completion_future.set(source_client.header_completion(id).fuse());
+			} else if let Some(header) = sync.headers().header(HeaderStatus::Extra) {
+				let id = header.id();
+				log::debug!(
+					target: "bridge",
+					"Retrieving extra data for header: {:?}",
+					id,
+				);
+				source_extra_future.set(source_client.header_extra(id, header.clone()).fuse());
+			} else if let Some(header) = sync.select_orphan_header_to_download() {
+				// for Orphan we actually ask for parent' header
+				let parent_id = header.parent_id();
+
+				// if we have end up with orphan header#0, then we are misconfigured
+				if parent_id.0.is_zero() {
+					log::error!(
+						target: "bridge",
+						"Misconfiguration. Genesis {} header is considered orphan by {} node",
+						P::SOURCE_NAME,
+						P::TARGET_NAME,
+					);
+					return Ok(());
+				}
+
+				log::debug!(
+					target: "bridge",
+					"Going to download orphan header from {} node: {:?}",
+					P::SOURCE_NAME,
+					parent_id,
+				);
+
+				source_orphan_header_future.set(source_client.header_by_hash(parent_id.1).fuse());
+			} else if let Some(id) = sync.select_new_header_to_download() {
+				log::debug!(
+					target: "bridge",
+					"Going to download new header from {} node: {:?}",
+					P::SOURCE_NAME,
+					id,
+				);
+
+				source_new_header_future.set(source_client.header_by_number(id).fuse());
+			} else {
+				source_client_is_online = true;
+			}
+		}
+	}
+}
+
+/// Print synchronization progress.
+fn print_sync_progress<P: HeadersSyncPipeline>(
+	progress_context: (Instant, Option<P::Number>, Option<P::Number>),
+	eth_sync: &HeadersSync<P>,
+) -> (Instant, Option<P::Number>, Option<P::Number>) {
+	let (prev_time, prev_best_header, prev_target_header) = progress_context;
+	let now_time = Instant::now();
+	let (now_best_header, now_target_header) = eth_sync.status();
+
+	let need_update = now_time - prev_time > Duration::from_secs(10)
+		|| match (prev_best_header, now_best_header) {
+			(Some(prev_best_header), Some(now_best_header)) => {
+				now_best_header.0.saturating_sub(prev_best_header) > 10.into()
+			}
+			_ => false,
+		};
+	if !need_update {
+		return (prev_time, prev_best_header, prev_target_header);
+	}
+
+	log::info!(
+		target: "bridge",
+		"Synced {:?} of {:?} headers",
+		now_best_header.map(|id| id.0),
+		now_target_header,
+	);
+	(now_time, (*now_best_header).map(|id| id.0), *now_target_header)
+}

polkadot/bridges/relays/headers/src/sync_loop_metrics.rs

@@ -0,0 +1,108 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Metrics for headers synchronization relay loop.
+
+use crate::sync::HeadersSync;
+use crate::sync_types::{HeaderStatus, HeadersSyncPipeline};
+
+use num_traits::Zero;
+use relay_utils::metrics::{metric_name, register, GaugeVec, Opts, PrometheusError, Registry, U64};
+
+/// Headers sync metrics.
+#[derive(Clone)]
+pub struct SyncLoopMetrics {
+	/// Best syncing headers at "source" and "target" nodes.
+	best_block_numbers: GaugeVec<U64>,
+	/// Number of headers in given states (see `HeaderStatus`).
+	blocks_in_state: GaugeVec<U64>,
+}
+
+impl SyncLoopMetrics {
+	/// Create and register headers loop metrics.
+	pub fn new(registry: &Registry, prefix: Option<&str>) -> Result<Self, PrometheusError> {
+		Ok(SyncLoopMetrics {
+			best_block_numbers: register(
+				GaugeVec::new(
+					Opts::new(
+						metric_name(prefix, "best_block_numbers"),
+						"Best block numbers on source and target nodes",
+					),
+					&["node"],
+				)?,
+				registry,
+			)?,
+			blocks_in_state: register(
+				GaugeVec::new(
+					Opts::new(
+						metric_name(prefix, "blocks_in_state"),
+						"Number of blocks in given state",
+					),
+					&["state"],
+				)?,
+				registry,
+			)?,
+		})
+	}
+}
+
+impl SyncLoopMetrics {
+	/// Update best block number at source.
+	pub fn update_best_block_at_source<Number: Into<u64>>(&self, source_best_number: Number) {
+		self.best_block_numbers
+			.with_label_values(&["source"])
+			.set(source_best_number.into());
+	}
+
+	/// Update best block number at target.
+	pub fn update_best_block_at_target<Number: Into<u64>>(&self, target_best_number: Number) {
+		self.best_block_numbers
+			.with_label_values(&["target"])
+			.set(target_best_number.into());
+	}
+
+	/// Update metrics.
+	pub fn update<P: HeadersSyncPipeline>(&self, sync: &HeadersSync<P>) {
+		let headers = sync.headers();
+		let source_best_number = sync.source_best_number().unwrap_or_else(Zero::zero);
+		let target_best_number = sync.target_best_header().map(|id| id.0).unwrap_or_else(Zero::zero);
+
+		self.update_best_block_at_source(source_best_number);
+		self.update_best_block_at_target(target_best_number);
+
+		self.blocks_in_state
+			.with_label_values(&["maybe_orphan"])
+			.set(headers.headers_in_status(HeaderStatus::MaybeOrphan) as _);
+		self.blocks_in_state
+			.with_label_values(&["orphan"])
+			.set(headers.headers_in_status(HeaderStatus::Orphan) as _);
+		self.blocks_in_state
+			.with_label_values(&["maybe_extra"])
+			.set(headers.headers_in_status(HeaderStatus::MaybeExtra) as _);
+		self.blocks_in_state
+			.with_label_values(&["extra"])
+			.set(headers.headers_in_status(HeaderStatus::Extra) as _);
+		self.blocks_in_state
+			.with_label_values(&["ready"])
+			.set(headers.headers_in_status(HeaderStatus::Ready) as _);
+		self.blocks_in_state
+			.with_label_values(&["incomplete"])
+			.set(headers.headers_in_status(HeaderStatus::Incomplete) as _);
+		self.blocks_in_state
+			.with_label_values(&["submitted"])
+			.set(headers.headers_in_status(HeaderStatus::Submitted) as _);
+	}
+}

polkadot/bridges/relays/headers/src/sync_loop_tests.rs

@@ -0,0 +1,594 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+#![cfg(test)]
+
+use crate::sync_loop::{run, SourceClient, TargetClient};
+use crate::sync_types::{HeadersSyncPipeline, QueuedHeader, SourceHeader, SubmittedHeaders};
+
+use async_trait::async_trait;
+use backoff::backoff::Backoff;
+use futures::{future::FutureExt, stream::StreamExt};
+use parking_lot::Mutex;
+use relay_utils::{
+	metrics::MetricsParams, process_future_result, relay_loop::Client as RelayClient, retry_backoff, HeaderId,
+	MaybeConnectionError,
+};
+use std::{
+	collections::{HashMap, HashSet},
+	sync::Arc,
+	time::Duration,
+};
+
+pub type TestNumber = u64;
+pub type TestHash = u64;
+pub type TestHeaderId = HeaderId<TestHash, TestNumber>;
+pub type TestExtra = u64;
+pub type TestCompletion = u64;
+pub type TestQueuedHeader = QueuedHeader<TestHeadersSyncPipeline>;
+
+#[derive(Default, Debug, Clone, PartialEq)]
+pub struct TestHeader {
+	pub hash: TestHash,
+	pub number: TestNumber,
+	pub parent_hash: TestHash,
+}
+
+impl SourceHeader<TestHash, TestNumber> for TestHeader {
+	fn id(&self) -> TestHeaderId {
+		HeaderId(self.number, self.hash)
+	}
+
+	fn parent_id(&self) -> TestHeaderId {
+		HeaderId(self.number - 1, self.parent_hash)
+	}
+}
+
+#[derive(Debug, Clone)]
+struct TestError(bool);
+
+impl MaybeConnectionError for TestError {
+	fn is_connection_error(&self) -> bool {
+		self.0
+	}
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct TestHeadersSyncPipeline;
+
+impl HeadersSyncPipeline for TestHeadersSyncPipeline {
+	const SOURCE_NAME: &'static str = "Source";
+	const TARGET_NAME: &'static str = "Target";
+
+	type Hash = TestHash;
+	type Number = TestNumber;
+	type Header = TestHeader;
+	type Extra = TestExtra;
+	type Completion = TestCompletion;
+
+	fn estimate_size(_: &TestQueuedHeader) -> usize {
+		0
+	}
+}
+
+enum SourceMethod {
+	BestBlockNumber,
+	HeaderByHash(TestHash),
+	HeaderByNumber(TestNumber),
+	HeaderCompletion(TestHeaderId),
+	HeaderExtra(TestHeaderId, TestQueuedHeader),
+}
+
+#[derive(Clone)]
+struct Source {
+	data: Arc<Mutex<SourceData>>,
+	on_method_call: Arc<dyn Fn(SourceMethod, &mut SourceData) + Send + Sync>,
+}
+
+struct SourceData {
+	best_block_number: Result<TestNumber, TestError>,
+	header_by_hash: HashMap<TestHash, TestHeader>,
+	header_by_number: HashMap<TestNumber, TestHeader>,
+	provides_completion: bool,
+	provides_extra: bool,
+}
+
+impl Source {
+	pub fn new(
+		best_block_id: TestHeaderId,
+		headers: Vec<(bool, TestHeader)>,
+		on_method_call: impl Fn(SourceMethod, &mut SourceData) + Send + Sync + 'static,
+	) -> Self {
+		Source {
+			data: Arc::new(Mutex::new(SourceData {
+				best_block_number: Ok(best_block_id.0),
+				header_by_hash: headers
+					.iter()
+					.map(|(_, header)| (header.hash, header.clone()))
+					.collect(),
+				header_by_number: headers
+					.iter()
+					.filter_map(|(is_canonical, header)| {
+						if *is_canonical {
+							Some((header.hash, header.clone()))
+						} else {
+							None
+						}
+					})
+					.collect(),
+				provides_completion: true,
+				provides_extra: true,
+			})),
+			on_method_call: Arc::new(on_method_call),
+		}
+	}
+}
+
+#[async_trait]
+impl RelayClient for Source {
+	type Error = TestError;
+
+	async fn reconnect(&mut self) -> Result<(), TestError> {
+		unimplemented!()
+	}
+}
+
+#[async_trait]
+impl SourceClient<TestHeadersSyncPipeline> for Source {
+	async fn best_block_number(&self) -> Result<TestNumber, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(SourceMethod::BestBlockNumber, &mut *data);
+		data.best_block_number.clone()
+	}
+
+	async fn header_by_hash(&self, hash: TestHash) -> Result<TestHeader, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(SourceMethod::HeaderByHash(hash), &mut *data);
+		data.header_by_hash.get(&hash).cloned().ok_or(TestError(false))
+	}
+
+	async fn header_by_number(&self, number: TestNumber) -> Result<TestHeader, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(SourceMethod::HeaderByNumber(number), &mut *data);
+		data.header_by_number.get(&number).cloned().ok_or(TestError(false))
+	}
+
+	async fn header_completion(&self, id: TestHeaderId) -> Result<(TestHeaderId, Option<TestCompletion>), TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(SourceMethod::HeaderCompletion(id), &mut *data);
+		if data.provides_completion {
+			Ok((id, Some(test_completion(id))))
+		} else {
+			Ok((id, None))
+		}
+	}
+
+	async fn header_extra(
+		&self,
+		id: TestHeaderId,
+		header: TestQueuedHeader,
+	) -> Result<(TestHeaderId, TestExtra), TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(SourceMethod::HeaderExtra(id, header), &mut *data);
+		if data.provides_extra {
+			Ok((id, test_extra(id)))
+		} else {
+			Err(TestError(false))
+		}
+	}
+}
+
+enum TargetMethod {
+	BestHeaderId,
+	IsKnownHeader(TestHeaderId),
+	SubmitHeaders(Vec<TestQueuedHeader>),
+	IncompleteHeadersIds,
+	CompleteHeader(TestHeaderId, TestCompletion),
+	RequiresExtra(TestQueuedHeader),
+}
+
+#[derive(Clone)]
+struct Target {
+	data: Arc<Mutex<TargetData>>,
+	on_method_call: Arc<dyn Fn(TargetMethod, &mut TargetData) + Send + Sync>,
+}
+
+struct TargetData {
+	best_header_id: Result<TestHeaderId, TestError>,
+	is_known_header_by_hash: HashMap<TestHash, bool>,
+	submitted_headers: HashMap<TestHash, TestQueuedHeader>,
+	submit_headers_result: Option<SubmittedHeaders<TestHeaderId, TestError>>,
+	completed_headers: HashMap<TestHash, TestCompletion>,
+	requires_completion: bool,
+	requires_extra: bool,
+}
+
+impl Target {
+	pub fn new(
+		best_header_id: TestHeaderId,
+		headers: Vec<TestHeaderId>,
+		on_method_call: impl Fn(TargetMethod, &mut TargetData) + Send + Sync + 'static,
+	) -> Self {
+		Target {
+			data: Arc::new(Mutex::new(TargetData {
+				best_header_id: Ok(best_header_id),
+				is_known_header_by_hash: headers.iter().map(|header| (header.1, true)).collect(),
+				submitted_headers: HashMap::new(),
+				submit_headers_result: None,
+				completed_headers: HashMap::new(),
+				requires_completion: false,
+				requires_extra: false,
+			})),
+			on_method_call: Arc::new(on_method_call),
+		}
+	}
+}
+
+#[async_trait]
+impl RelayClient for Target {
+	type Error = TestError;
+
+	async fn reconnect(&mut self) -> Result<(), TestError> {
+		unimplemented!()
+	}
+}
+
+#[async_trait]
+impl TargetClient<TestHeadersSyncPipeline> for Target {
+	async fn best_header_id(&self) -> Result<TestHeaderId, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(TargetMethod::BestHeaderId, &mut *data);
+		data.best_header_id.clone()
+	}
+
+	async fn is_known_header(&self, id: TestHeaderId) -> Result<(TestHeaderId, bool), TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(TargetMethod::IsKnownHeader(id), &mut *data);
+		data.is_known_header_by_hash
+			.get(&id.1)
+			.cloned()
+			.map(|is_known_header| Ok((id, is_known_header)))
+			.unwrap_or(Ok((id, false)))
+	}
+
+	async fn submit_headers(&self, headers: Vec<TestQueuedHeader>) -> SubmittedHeaders<TestHeaderId, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(TargetMethod::SubmitHeaders(headers.clone()), &mut *data);
+		data.submitted_headers
+			.extend(headers.iter().map(|header| (header.id().1, header.clone())));
+		data.submit_headers_result.take().expect("test must accept headers")
+	}
+
+	async fn incomplete_headers_ids(&self) -> Result<HashSet<TestHeaderId>, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(TargetMethod::IncompleteHeadersIds, &mut *data);
+		if data.requires_completion {
+			Ok(data
+				.submitted_headers
+				.iter()
+				.filter(|(hash, _)| !data.completed_headers.contains_key(hash))
+				.map(|(_, header)| header.id())
+				.collect())
+		} else {
+			Ok(HashSet::new())
+		}
+	}
+
+	async fn complete_header(&self, id: TestHeaderId, completion: TestCompletion) -> Result<TestHeaderId, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(TargetMethod::CompleteHeader(id, completion), &mut *data);
+		data.completed_headers.insert(id.1, completion);
+		Ok(id)
+	}
+
+	async fn requires_extra(&self, header: TestQueuedHeader) -> Result<(TestHeaderId, bool), TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(TargetMethod::RequiresExtra(header.clone()), &mut *data);
+		if data.requires_extra {
+			Ok((header.id(), true))
+		} else {
+			Ok((header.id(), false))
+		}
+	}
+}
+
+fn test_tick() -> Duration {
+	// in ideal world that should have been Duration::from_millis(0), because we do not want
+	// to sleep in tests at all, but that could lead to `select! {}` always waking on tick
+	// => not doing actual job
+	Duration::from_millis(10)
+}
+
+fn test_id(number: TestNumber) -> TestHeaderId {
+	HeaderId(number, number)
+}
+
+fn test_header(number: TestNumber) -> TestHeader {
+	let id = test_id(number);
+	TestHeader {
+		hash: id.1,
+		number: id.0,
+		parent_hash: if number == 0 {
+			TestHash::default()
+		} else {
+			test_id(number - 1).1
+		},
+	}
+}
+
+fn test_forked_id(number: TestNumber, forked_from: TestNumber) -> TestHeaderId {
+	const FORK_OFFSET: TestNumber = 1000;
+
+	if number == forked_from {
+		HeaderId(number, number)
+	} else {
+		HeaderId(number, FORK_OFFSET + number)
+	}
+}
+
+fn test_forked_header(number: TestNumber, forked_from: TestNumber) -> TestHeader {
+	let id = test_forked_id(number, forked_from);
+	TestHeader {
+		hash: id.1,
+		number: id.0,
+		parent_hash: if number == 0 {
+			TestHash::default()
+		} else {
+			test_forked_id(number - 1, forked_from).1
+		},
+	}
+}
+
+fn test_completion(id: TestHeaderId) -> TestCompletion {
+	id.0
+}
+
+fn test_extra(id: TestHeaderId) -> TestExtra {
+	id.0
+}
+
+fn source_reject_completion(method: &SourceMethod) {
+	if let SourceMethod::HeaderCompletion(_) = method {
+		unreachable!("HeaderCompletion request is not expected")
+	}
+}
+
+fn source_reject_extra(method: &SourceMethod) {
+	if let SourceMethod::HeaderExtra(_, _) = method {
+		unreachable!("HeaderExtra request is not expected")
+	}
+}
+
+fn target_accept_all_headers(method: &TargetMethod, data: &mut TargetData, requires_extra: bool) {
+	if let TargetMethod::SubmitHeaders(ref submitted) = method {
+		assert_eq!(submitted.iter().all(|header| header.extra().is_some()), requires_extra,);
+
+		data.submit_headers_result = Some(SubmittedHeaders {
+			submitted: submitted.iter().map(|header| header.id()).collect(),
+			..Default::default()
+		});
+	}
+}
+
+fn target_signal_exit_when_header_submitted(
+	method: &TargetMethod,
+	header_id: TestHeaderId,
+	exit_signal: &futures::channel::mpsc::UnboundedSender<()>,
+) {
+	if let TargetMethod::SubmitHeaders(ref submitted) = method {
+		if submitted.iter().any(|header| header.id() == header_id) {
+			exit_signal.unbounded_send(()).unwrap();
+		}
+	}
+}
+
+fn target_signal_exit_when_header_completed(
+	method: &TargetMethod,
+	header_id: TestHeaderId,
+	exit_signal: &futures::channel::mpsc::UnboundedSender<()>,
+) {
+	if let TargetMethod::CompleteHeader(completed_id, _) = method {
+		if *completed_id == header_id {
+			exit_signal.unbounded_send(()).unwrap();
+		}
+	}
+}
+
+fn run_backoff_test(result: Result<(), TestError>) -> (Duration, Duration) {
+	let mut backoff = retry_backoff();
+
+	// no randomness in tests (otherwise intervals may overlap => asserts are failing)
+	backoff.randomization_factor = 0f64;
+
+	// increase backoff's current interval
+	let interval1 = backoff.next_backoff().unwrap();
+	let interval2 = backoff.next_backoff().unwrap();
+	assert!(interval2 > interval1);
+
+	// successful future result leads to backoff's reset
+	let go_offline_future = futures::future::Fuse::terminated();
+	futures::pin_mut!(go_offline_future);
+
+	process_future_result(
+		result,
+		&mut backoff,
+		|_| {},
+		&mut go_offline_future,
+		async_std::task::sleep,
+		|| "Test error".into(),
+	);
+
+	(interval2, backoff.next_backoff().unwrap())
+}
+
+#[test]
+fn process_future_result_resets_backoff_on_success() {
+	let (interval2, interval_after_reset) = run_backoff_test(Ok(()));
+	assert!(interval2 > interval_after_reset);
+}
+
+#[test]
+fn process_future_result_resets_backoff_on_connection_error() {
+	let (interval2, interval_after_reset) = run_backoff_test(Err(TestError(true)));
+	assert!(interval2 > interval_after_reset);
+}
+
+#[test]
+fn process_future_result_does_not_reset_backoff_on_non_connection_error() {
+	let (interval2, interval_after_reset) = run_backoff_test(Err(TestError(false)));
+	assert!(interval2 < interval_after_reset);
+}
+
+struct SyncLoopTestParams {
+	best_source_header: TestHeader,
+	headers_on_source: Vec<(bool, TestHeader)>,
+	best_target_header: TestHeader,
+	headers_on_target: Vec<TestHeader>,
+	target_requires_extra: bool,
+	target_requires_completion: bool,
+	stop_at: TestHeaderId,
+}
+
+fn run_sync_loop_test(params: SyncLoopTestParams) {
+	let (exit_sender, exit_receiver) = futures::channel::mpsc::unbounded();
+	let target_requires_extra = params.target_requires_extra;
+	let target_requires_completion = params.target_requires_completion;
+	let stop_at = params.stop_at;
+	let source = Source::new(
+		params.best_source_header.id(),
+		params.headers_on_source,
+		move |method, _| {
+			if !target_requires_extra {
+				source_reject_extra(&method);
+			}
+			if !target_requires_completion {
+				source_reject_completion(&method);
+			}
+		},
+	);
+	let target = Target::new(
+		params.best_target_header.id(),
+		params.headers_on_target.into_iter().map(|header| header.id()).collect(),
+		move |method, data| {
+			target_accept_all_headers(&method, data, target_requires_extra);
+			if target_requires_completion {
+				target_signal_exit_when_header_completed(&method, stop_at, &exit_sender);
+			} else {
+				target_signal_exit_when_header_submitted(&method, stop_at, &exit_sender);
+			}
+		},
+	);
+	target.data.lock().requires_extra = target_requires_extra;
+	target.data.lock().requires_completion = target_requires_completion;
+
+	let _ = async_std::task::block_on(run(
+		source,
+		test_tick(),
+		target,
+		test_tick(),
+		(),
+		crate::sync::tests::default_sync_params(),
+		MetricsParams::disabled(),
+		exit_receiver.into_future().map(|(_, _)| ()),
+	));
+}
+
+#[test]
+fn sync_loop_is_able_to_synchronize_single_header() {
+	run_sync_loop_test(SyncLoopTestParams {
+		best_source_header: test_header(1),
+		headers_on_source: vec![(true, test_header(1))],
+		best_target_header: test_header(0),
+		headers_on_target: vec![test_header(0)],
+		target_requires_extra: false,
+		target_requires_completion: false,
+		stop_at: test_id(1),
+	});
+}
+
+#[test]
+fn sync_loop_is_able_to_synchronize_single_header_with_extra() {
+	run_sync_loop_test(SyncLoopTestParams {
+		best_source_header: test_header(1),
+		headers_on_source: vec![(true, test_header(1))],
+		best_target_header: test_header(0),
+		headers_on_target: vec![test_header(0)],
+		target_requires_extra: true,
+		target_requires_completion: false,
+		stop_at: test_id(1),
+	});
+}
+
+#[test]
+fn sync_loop_is_able_to_synchronize_single_header_with_completion() {
+	run_sync_loop_test(SyncLoopTestParams {
+		best_source_header: test_header(1),
+		headers_on_source: vec![(true, test_header(1))],
+		best_target_header: test_header(0),
+		headers_on_target: vec![test_header(0)],
+		target_requires_extra: false,
+		target_requires_completion: true,
+		stop_at: test_id(1),
+	});
+}
+
+#[test]
+fn sync_loop_is_able_to_reorganize_from_shorter_fork() {
+	run_sync_loop_test(SyncLoopTestParams {
+		best_source_header: test_header(3),
+		headers_on_source: vec![
+			(true, test_header(1)),
+			(true, test_header(2)),
+			(true, test_header(3)),
+			(false, test_forked_header(1, 0)),
+			(false, test_forked_header(2, 0)),
+		],
+		best_target_header: test_forked_header(2, 0),
+		headers_on_target: vec![test_header(0), test_forked_header(1, 0), test_forked_header(2, 0)],
+		target_requires_extra: false,
+		target_requires_completion: false,
+		stop_at: test_id(3),
+	});
+}
+
+#[test]
+fn sync_loop_is_able_to_reorganize_from_longer_fork() {
+	run_sync_loop_test(SyncLoopTestParams {
+		best_source_header: test_header(3),
+		headers_on_source: vec![
+			(true, test_header(1)),
+			(true, test_header(2)),
+			(true, test_header(3)),
+			(false, test_forked_header(1, 0)),
+			(false, test_forked_header(2, 0)),
+			(false, test_forked_header(3, 0)),
+			(false, test_forked_header(4, 0)),
+			(false, test_forked_header(5, 0)),
+		],
+		best_target_header: test_forked_header(5, 0),
+		headers_on_target: vec![
+			test_header(0),
+			test_forked_header(1, 0),
+			test_forked_header(2, 0),
+			test_forked_header(3, 0),
+			test_forked_header(4, 0),
+			test_forked_header(5, 0),
+		],
+		target_requires_extra: false,
+		target_requires_completion: false,
+		stop_at: test_id(3),
+	});
+}

polkadot/bridges/relays/headers/src/sync_types.rs

@@ -0,0 +1,189 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Types that are used by headers synchronization components.
+
+use relay_utils::{format_ids, HeaderId};
+use std::{ops::Deref, sync::Arc};
+
+/// Ethereum header synchronization status.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum HeaderStatus {
+	/// Header is unknown.
+	Unknown,
+	/// Header is in MaybeOrphan queue.
+	MaybeOrphan,
+	/// Header is in Orphan queue.
+	Orphan,
+	/// Header is in MaybeExtra queue.
+	MaybeExtra,
+	/// Header is in Extra queue.
+	Extra,
+	/// Header is in Ready queue.
+	Ready,
+	/// Header is in Incomplete queue.
+	Incomplete,
+	/// Header has been recently submitted to the target node.
+	Submitted,
+	/// Header is known to the target node.
+	Synced,
+}
+
+/// Headers synchronization pipeline.
+pub trait HeadersSyncPipeline: Clone + Send + Sync {
+	/// Name of the headers source.
+	const SOURCE_NAME: &'static str;
+	/// Name of the headers target.
+	const TARGET_NAME: &'static str;
+
+	/// Headers we're syncing are identified by this hash.
+	type Hash: Eq + Clone + Copy + Send + Sync + std::fmt::Debug + std::fmt::Display + std::hash::Hash;
+	/// Headers we're syncing are identified by this number.
+	type Number: relay_utils::BlockNumberBase;
+	/// Type of header that we're syncing.
+	type Header: SourceHeader<Self::Hash, Self::Number>;
+	/// Type of extra data for the header that we're receiving from the source node:
+	/// 1) extra data is required for some headers;
+	/// 2) target node may answer if it'll require extra data before header is submitted;
+	/// 3) extra data available since the header creation time;
+	/// 4) header and extra data are submitted in single transaction.
+	///
+	/// Example: Ethereum transactions receipts.
+	type Extra: Clone + Send + Sync + PartialEq + std::fmt::Debug;
+	/// Type of data required to 'complete' header that we're receiving from the source node:
+	/// 1) completion data is required for some headers;
+	/// 2) target node can't answer if it'll require completion data before header is accepted;
+	/// 3) completion data may be generated after header generation;
+	/// 4) header and completion data are submitted in separate transactions.
+	///
+	/// Example: Substrate GRANDPA justifications.
+	type Completion: Clone + Send + Sync + std::fmt::Debug;
+
+	/// Function used to estimate size of target-encoded header.
+	fn estimate_size(source: &QueuedHeader<Self>) -> usize;
+}
+
+/// A HeaderId for `HeaderSyncPipeline`.
+pub type HeaderIdOf<P> = HeaderId<<P as HeadersSyncPipeline>::Hash, <P as HeadersSyncPipeline>::Number>;
+
+/// Header that we're receiving from source node.
+pub trait SourceHeader<Hash, Number>: Clone + std::fmt::Debug + PartialEq + Send + Sync {
+	/// Returns ID of header.
+	fn id(&self) -> HeaderId<Hash, Number>;
+	/// Returns ID of parent header.
+	///
+	/// Panics if called for genesis header.
+	fn parent_id(&self) -> HeaderId<Hash, Number>;
+}
+
+/// Header how it's stored in the synchronization queue.
+#[derive(Clone, Debug, PartialEq)]
+pub struct QueuedHeader<P: HeadersSyncPipeline>(Arc<QueuedHeaderData<P>>);
+
+impl<P: HeadersSyncPipeline> QueuedHeader<P> {
+	/// Creates new queued header.
+	pub fn new(header: P::Header) -> Self {
+		QueuedHeader(Arc::new(QueuedHeaderData { header, extra: None }))
+	}
+
+	/// Set associated extra data.
+	pub fn set_extra(self, extra: P::Extra) -> Self {
+		QueuedHeader(Arc::new(QueuedHeaderData {
+			header: Arc::try_unwrap(self.0)
+				.map(|data| data.header)
+				.unwrap_or_else(|data| data.header.clone()),
+			extra: Some(extra),
+		}))
+	}
+}
+
+impl<P: HeadersSyncPipeline> Deref for QueuedHeader<P> {
+	type Target = QueuedHeaderData<P>;
+
+	fn deref(&self) -> &Self::Target {
+		&self.0
+	}
+}
+
+/// Header how it's stored in the synchronization queue.
+#[derive(Clone, Debug, Default, PartialEq)]
+pub struct QueuedHeaderData<P: HeadersSyncPipeline> {
+	header: P::Header,
+	extra: Option<P::Extra>,
+}
+
+impl<P: HeadersSyncPipeline> QueuedHeader<P> {
+	/// Returns ID of header.
+	pub fn id(&self) -> HeaderId<P::Hash, P::Number> {
+		self.header.id()
+	}
+
+	/// Returns ID of parent header.
+	pub fn parent_id(&self) -> HeaderId<P::Hash, P::Number> {
+		self.header.parent_id()
+	}
+
+	/// Returns reference to header.
+	pub fn header(&self) -> &P::Header {
+		&self.header
+	}
+
+	/// Returns reference to associated extra data.
+	pub fn extra(&self) -> &Option<P::Extra> {
+		&self.extra
+	}
+}
+
+/// Headers submission result.
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+pub struct SubmittedHeaders<Id, Error> {
+	/// IDs of headers that have been submitted to target node.
+	pub submitted: Vec<Id>,
+	/// IDs of incomplete headers. These headers were submitted (so this id is also in `submitted` vec),
+	/// but all descendants are not.
+	pub incomplete: Vec<Id>,
+	/// IDs of ignored headers that we have decided not to submit (they're either rejected by
+	/// target node immediately, or they're descendants of incomplete headers).
+	pub rejected: Vec<Id>,
+	/// Fatal target node error, if it has occured during submission.
+	pub fatal_error: Option<Error>,
+}
+
+impl<Id, Error> Default for SubmittedHeaders<Id, Error> {
+	fn default() -> Self {
+		SubmittedHeaders {
+			submitted: Vec::new(),
+			incomplete: Vec::new(),
+			rejected: Vec::new(),
+			fatal_error: None,
+		}
+	}
+}
+
+impl<Id: std::fmt::Debug, Error> std::fmt::Display for SubmittedHeaders<Id, Error> {
+	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+		let submitted = format_ids(self.submitted.iter());
+		let incomplete = format_ids(self.incomplete.iter());
+		let rejected = format_ids(self.rejected.iter());
+
+		write!(
+			f,
+			"Submitted: {}, Incomplete: {}, Rejected: {}",
+			submitted, incomplete, rejected
+		)
+	}
+}