fix: Complete snowbridge pezpallet rebrand and critical bug fixes

- snowbridge-pezpallet-* → pezsnowbridge-pezpallet-* (201 refs)
- pallet/ directories → pezpallet/ (4 locations)
- Fixed pezpallet.rs self-include recursion bug
- Fixed sc-chain-spec hardcoded crate name in derive macro
- Reverted .pezpallet_by_name() to .pallet_by_name() (subxt API)
- Added BizinikiwiConfig type alias for zombienet tests
- Deleted obsolete session state files

Verified: pezsnowbridge-pezpallet-*, pezpallet-staking,
pezpallet-staking-async, pezframe-benchmarking-cli all pass cargo check

pezbridges/relays/client-bizinikiwi/Cargo.toml

@@ -0,0 +1,84 @@
+[package]
+name = "relay-bizinikiwi-client"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+repository.workspace = true
+publish = false
+description = "Pezkuwi SDK component: relay bizinikiwi client"
+documentation = "https://docs.rs/relay-bizinikiwi-client"
+homepage = { workspace = true }
+
+[lints]
+workspace = true
+
+[dependencies]
+async-std = { features = ["attributes"], workspace = true }
+async-trait = { workspace = true }
+codec = { workspace = true, default-features = true }
+futures = { workspace = true }
+jsonrpsee = { features = ["macros", "ws-client"], workspace = true }
+num-traits = { workspace = true, default-features = true }
+quick_cache = { workspace = true }
+rand = { workspace = true, default-features = true }
+scale-info = { features = [
+  "derive",
+], workspace = true, default-features = true }
+serde_json = { workspace = true }
+thiserror = { workspace = true }
+tokio = { features = [
+  "rt-multi-thread",
+], workspace = true, default-features = true }
+tracing = { workspace = true }
+
+# Bridge dependencies
+bp-header-pez-chain = { workspace = true, default-features = true }
+bp-messages = { workspace = true, default-features = true }
+bp-pezkuwi-core = { workspace = true, default-features = true }
+pezbp-runtime = { workspace = true, default-features = true }
+pez-finality-relay = { workspace = true }
+relay-utils = { workspace = true }
+
+# Bizinikiwi Dependencies
+pezframe-support = { workspace = true, default-features = true }
+pezpallet-transaction-payment = { workspace = true, default-features = true }
+pezpallet-transaction-payment-rpc-runtime-api = { workspace = true, default-features = true }
+pezpallet-utility = { workspace = true, default-features = true }
+pezsc-chain-spec = { workspace = true, default-features = true }
+pezsc-rpc-api = { workspace = true, default-features = true }
+pezsc-transaction-pool-api = { workspace = true, default-features = true }
+pezsp-consensus-grandpa = { workspace = true, default-features = true }
+pezsp-core = { workspace = true, default-features = true }
+pezsp-rpc = { workspace = true, default-features = true }
+pezsp-runtime = { workspace = true, default-features = true }
+pezsp-std = { workspace = true, default-features = true }
+pezsp-trie = { workspace = true, default-features = true }
+pezsp-version = { workspace = true, default-features = true }
+
+# Pezkuwi Dependencies
+xcm = { workspace = true, default-features = true }
+
+[features]
+default = []
+test-helpers = []
+runtime-benchmarks = [
+  "bp-header-pez-chain/runtime-benchmarks",
+  "bp-messages/runtime-benchmarks",
+  "bp-pezkuwi-core/runtime-benchmarks",
+  "pezbp-runtime/runtime-benchmarks",
+  "pez-finality-relay/runtime-benchmarks",
+  "pezframe-support/runtime-benchmarks",
+  "pezpallet-transaction-payment-rpc-runtime-api/runtime-benchmarks",
+  "pezpallet-transaction-payment/runtime-benchmarks",
+  "pezpallet-utility/runtime-benchmarks",
+  "relay-utils/runtime-benchmarks",
+  "pezsc-chain-spec/runtime-benchmarks",
+  "pezsc-rpc-api/runtime-benchmarks",
+  "pezsc-transaction-pool-api/runtime-benchmarks",
+  "pezsp-consensus-grandpa/runtime-benchmarks",
+  "pezsp-runtime/runtime-benchmarks",
+  "pezsp-trie/runtime-benchmarks",
+  "pezsp-version/runtime-benchmarks",
+  "xcm/runtime-benchmarks",
+]

pezbridges/relays/client-bizinikiwi/src/calls.rs

@@ -0,0 +1,59 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Basic runtime calls.
+
+use codec::{Decode, Encode};
+use scale_info::TypeInfo;
+use pezsp_std::{boxed::Box, vec::Vec};
+
+use xcm::{VersionedLocation, VersionedXcm};
+
+/// A minimized version of `pezframe-system::Call` that can be used without a runtime.
+#[derive(Encode, Decode, Debug, PartialEq, Eq, Clone, TypeInfo)]
+#[allow(non_camel_case_types)]
+pub enum SystemCall {
+	/// `pezframe-system::Call::remark`
+	#[codec(index = 1)]
+	remark(Vec<u8>),
+}
+
+/// A minimized version of `pezpallet-utility::Call` that can be used without a runtime.
+#[derive(Encode, Decode, Debug, PartialEq, Eq, Clone, TypeInfo)]
+#[allow(non_camel_case_types)]
+pub enum UtilityCall<Call> {
+	/// `pezpallet-utility::Call::batch_all`
+	#[codec(index = 2)]
+	batch_all(Vec<Call>),
+}
+
+/// A minimized version of `pezpallet-sudo::Call` that can be used without a runtime.
+#[derive(Encode, Decode, Debug, PartialEq, Eq, Clone, TypeInfo)]
+#[allow(non_camel_case_types)]
+pub enum SudoCall<Call> {
+	/// `pezpallet-sudo::Call::sudo`
+	#[codec(index = 0)]
+	sudo(Box<Call>),
+}
+
+/// A minimized version of `pezpallet-xcm::Call`, that can be used without a runtime.
+#[derive(Encode, Decode, Debug, PartialEq, Eq, Clone, TypeInfo)]
+#[allow(non_camel_case_types)]
+pub enum XcmCall {
+	/// `pezpallet-xcm::Call::send`
+	#[codec(index = 0)]
+	send(Box<VersionedLocation>, Box<VersionedXcm<()>>),
+}

pezbridges/relays/client-bizinikiwi/src/chain.rs

@@ -0,0 +1,305 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::calls::UtilityCall;
+
+use crate::SimpleRuntimeVersion;
+use bp_header_pez_chain::ChainWithGrandpa as ChainWithGrandpaBase;
+use bp_messages::ChainWithMessages as ChainWithMessagesBase;
+use pezbp_runtime::{
+	Chain as ChainBase, EncodedOrDecodedCall, HashOf, Teyrchain as TeyrchainBase, TransactionEra,
+	TransactionEraOf, UnderlyingChainProvider,
+};
+use codec::{Codec, Decode, Encode, MaxEncodedLen};
+use pezframe_support::Parameter;
+use jsonrpsee::core::{DeserializeOwned, Serialize};
+use num_traits::Zero;
+use pezsc_transaction_pool_api::TransactionStatus;
+use scale_info::TypeInfo;
+use pezsp_core::{storage::StorageKey, Pair};
+use pezsp_runtime::{
+	generic::SignedBlock,
+	traits::{AtLeast32BitUnsigned, Block as BlockT, Member},
+	ConsensusEngineId, EncodedJustification,
+};
+use std::{fmt::Debug, time::Duration};
+
+/// Signed block type of given chain.
+pub type SignedBlockOf<C> = <C as Chain>::SignedBlock;
+
+/// Bizinikiwi-based chain from minimal relay-client point of view.
+pub trait Chain: ChainBase + Clone {
+	/// Chain name.
+	const NAME: &'static str;
+	/// Name of the runtime API method that is returning best known finalized header number
+	/// and hash (as tuple).
+	///
+	/// Keep in mind that this method is normally provided by the other chain, which is
+	/// bridged with this chain.
+	const BEST_FINALIZED_HEADER_ID_METHOD: &'static str;
+	/// Name of the runtime API method that is returning interval between source chain
+	/// headers that may be submitted for free to the target chain.
+	///
+	/// Keep in mind that this method is normally provided by the other chain, which is
+	/// bridged with this chain.
+	const FREE_HEADERS_INTERVAL_METHOD: &'static str;
+
+	/// Average block interval.
+	///
+	/// How often blocks are produced on that chain. It's suggested to set this value
+	/// to match the block time of the chain.
+	const AVERAGE_BLOCK_INTERVAL: Duration;
+
+	/// Block type.
+	type SignedBlock: Member + Serialize + DeserializeOwned + BlockWithJustification<Self::Header>;
+	/// The aggregated `Call` type.
+	type Call: Clone + Codec + Debug + Send + Sync;
+}
+
+/// Bridge-supported network definition.
+///
+/// Used to abstract away CLI commands.
+pub trait ChainWithRuntimeVersion: Chain {
+	/// Current version of the chain runtime, known to relay.
+	///
+	/// can be `None` if relay is not going to submit transactions to that chain.
+	const RUNTIME_VERSION: Option<SimpleRuntimeVersion>;
+}
+
+/// Bizinikiwi-based relay chain that supports teyrchains.
+///
+/// We assume that the teyrchains are supported using `runtime_teyrchains::paras` pezpallet.
+pub trait RelayChain: Chain {
+	/// Name of the `runtime_teyrchains::paras` pezpallet in the runtime of this chain.
+	const PARAS_PALLET_NAME: &'static str;
+	/// Name of the `pezpallet-bridge-teyrchains`, deployed at the **bridged** chain to sync
+	/// teyrchains of **this** chain.
+	const WITH_CHAIN_BRIDGE_TEYRCHAINS_PALLET_NAME: &'static str;
+}
+
+/// Bizinikiwi-based chain that is using direct GRANDPA finality from minimal relay-client point of
+/// view.
+///
+/// Keep in mind that teyrchains are relying on relay chain GRANDPA, so they should not implement
+/// this trait.
+pub trait ChainWithGrandpa: Chain + ChainWithGrandpaBase {
+	/// Name of the runtime API method that is returning the GRANDPA info associated with the
+	/// headers accepted by the `submit_finality_proofs` extrinsic in the queried block.
+	///
+	/// Keep in mind that this method is normally provided by the other chain, which is
+	/// bridged with this chain.
+	const SYNCED_HEADERS_GRANDPA_INFO_METHOD: &'static str;
+
+	/// The type of the key owner proof used by the grandpa engine.
+	type KeyOwnerProof: Decode + TypeInfo + Send;
+}
+
+/// Bizinikiwi-based teyrchain from minimal relay-client point of view.
+pub trait Teyrchain: Chain + TeyrchainBase {}
+
+impl<T> Teyrchain for T where T: UnderlyingChainProvider + Chain + TeyrchainBase {}
+
+/// Bizinikiwi-based chain with messaging support from minimal relay-client point of view.
+pub trait ChainWithMessages: Chain + ChainWithMessagesBase {
+	/// Name of the `To<ChainWithMessages>OutboundLaneApi::message_details` runtime API method.
+	/// The method is provided by the runtime that is bridged with this `ChainWithMessages`.
+	const TO_CHAIN_MESSAGE_DETAILS_METHOD: &'static str;
+
+	/// Name of the `From<ChainWithMessages>InboundLaneApi::message_details` runtime API method.
+	/// The method is provided by the runtime that is bridged with this `ChainWithMessages`.
+	const FROM_CHAIN_MESSAGE_DETAILS_METHOD: &'static str;
+}
+
+/// Call type used by the chain.
+pub type CallOf<C> = <C as Chain>::Call;
+/// Transaction status of the chain.
+pub type TransactionStatusOf<C> = TransactionStatus<HashOf<C>, HashOf<C>>;
+
+/// Bizinikiwi-based chain with `AccountData` generic argument of `pezframe_system::AccountInfo` set to
+/// the `pezpallet_balances::AccountData<Balance>`.
+pub trait ChainWithBalances: Chain {
+	/// Return runtime storage key for getting `pezframe_system::AccountInfo` of given account.
+	fn account_info_storage_key(account_id: &Self::AccountId) -> StorageKey;
+}
+
+/// Bizinikiwi-based chain with bridge relayers pezpallet as a reward ledger.
+pub trait ChainWithRewards: Chain {
+	/// Name of the bridge relayers pezpallet (used in `construct_runtime` macro call).
+	const WITH_CHAIN_RELAYERS_PALLET_NAME: Option<&'static str>;
+	/// Type of relayer reward balance.
+	type RewardBalance: AtLeast32BitUnsigned + Copy + Member + Parameter + MaxEncodedLen;
+	/// Reward discriminator type.
+	type Reward: Parameter + MaxEncodedLen + Send + Sync + Copy + Clone;
+
+	/// Return runtime storage key for getting `reward_kind` of given account.
+	fn account_reward_storage_key(
+		account_id: &Self::AccountId,
+		reward: impl Into<Self::Reward>,
+	) -> StorageKey;
+}
+
+/// SCALE-encoded extrinsic.
+pub type EncodedExtrinsic = Vec<u8>;
+
+/// Block with justification.
+pub trait BlockWithJustification<Header> {
+	/// Return block header.
+	fn header(&self) -> Header;
+	/// Return encoded block extrinsics.
+	fn extrinsics(&self) -> Vec<EncodedExtrinsic>;
+	/// Return block justification, if known.
+	fn justification(&self, engine_id: ConsensusEngineId) -> Option<&EncodedJustification>;
+}
+
+/// Transaction before it is signed.
+#[derive(Clone, Debug, PartialEq)]
+pub struct UnsignedTransaction<C: Chain> {
+	/// Runtime call of this transaction.
+	pub call: EncodedOrDecodedCall<C::Call>,
+	/// Transaction nonce.
+	pub nonce: C::Nonce,
+	/// Tip included into transaction.
+	pub tip: C::Balance,
+	/// Transaction era used by the chain.
+	pub era: TransactionEraOf<C>,
+}
+
+impl<C: Chain> UnsignedTransaction<C> {
+	/// Create new unsigned transaction with given call, nonce, era and zero tip.
+	pub fn new(call: EncodedOrDecodedCall<C::Call>, nonce: C::Nonce) -> Self {
+		Self { call, nonce, era: TransactionEra::Immortal, tip: Zero::zero() }
+	}
+
+	/// Convert to the transaction of the other compatible chain.
+	pub fn switch_chain<Other>(self) -> UnsignedTransaction<Other>
+	where
+		Other: Chain<
+			Nonce = C::Nonce,
+			Balance = C::Balance,
+			BlockNumber = C::BlockNumber,
+			Hash = C::Hash,
+		>,
+	{
+		UnsignedTransaction {
+			call: EncodedOrDecodedCall::Encoded(self.call.into_encoded()),
+			nonce: self.nonce,
+			tip: self.tip,
+			era: self.era,
+		}
+	}
+
+	/// Set transaction tip.
+	#[must_use]
+	pub fn tip(mut self, tip: C::Balance) -> Self {
+		self.tip = tip;
+		self
+	}
+
+	/// Set transaction era.
+	#[must_use]
+	pub fn era(mut self, era: TransactionEraOf<C>) -> Self {
+		self.era = era;
+		self
+	}
+}
+
+/// Account key pair used by transactions signing scheme.
+pub type AccountKeyPairOf<S> = <S as ChainWithTransactions>::AccountKeyPair;
+
+/// Bizinikiwi-based chain transactions signing scheme.
+pub trait ChainWithTransactions: Chain {
+	/// Type of key pairs used to sign transactions.
+	type AccountKeyPair: Pair + Clone + Send + Sync;
+	/// Signed transaction.
+	type SignedTransaction: Clone + Debug + Encode + Send + 'static;
+
+	/// Create transaction for given runtime call, signed by given account.
+	fn sign_transaction(
+		param: SignParam<Self>,
+		unsigned: UnsignedTransaction<Self>,
+	) -> Result<Self::SignedTransaction, crate::Error>
+	where
+		Self: Sized;
+}
+
+/// Sign transaction parameters
+pub struct SignParam<C: ChainWithTransactions> {
+	/// Version of the runtime specification.
+	pub spec_version: u32,
+	/// Transaction version
+	pub transaction_version: u32,
+	/// Hash of the genesis block.
+	pub genesis_hash: HashOf<C>,
+	/// Signer account
+	pub signer: AccountKeyPairOf<C>,
+}
+
+impl<Block: BlockT> BlockWithJustification<Block::Header> for SignedBlock<Block> {
+	fn header(&self) -> Block::Header {
+		self.block.header().clone()
+	}
+
+	fn extrinsics(&self) -> Vec<EncodedExtrinsic> {
+		self.block.extrinsics().iter().map(Encode::encode).collect()
+	}
+
+	fn justification(&self, engine_id: ConsensusEngineId) -> Option<&EncodedJustification> {
+		self.justifications.as_ref().and_then(|j| j.get(engine_id))
+	}
+}
+
+/// Trait that provides functionality defined inside `pezpallet-utility`
+pub trait UtilityPallet<C: Chain> {
+	/// Create batch call from given calls vector.
+	fn build_batch_call(calls: Vec<C::Call>) -> C::Call;
+}
+
+/// Structure that implements `UtilityPalletProvider` based on a full runtime.
+pub struct FullRuntimeUtilityPallet<R> {
+	_phantom: std::marker::PhantomData<R>,
+}
+
+impl<C, R> UtilityPallet<C> for FullRuntimeUtilityPallet<R>
+where
+	C: Chain,
+	R: pezpallet_utility::Config<RuntimeCall = C::Call>,
+	<R as pezpallet_utility::Config>::RuntimeCall: From<pezpallet_utility::Call<R>>,
+{
+	fn build_batch_call(calls: Vec<C::Call>) -> C::Call {
+		pezpallet_utility::Call::batch_all { calls }.into()
+	}
+}
+
+/// Structure that implements `UtilityPalletProvider` based on a call conversion.
+pub struct MockedRuntimeUtilityPallet<Call> {
+	_phantom: std::marker::PhantomData<Call>,
+}
+
+impl<C, Call> UtilityPallet<C> for MockedRuntimeUtilityPallet<Call>
+where
+	C: Chain,
+	C::Call: From<UtilityCall<C::Call>>,
+{
+	fn build_batch_call(calls: Vec<C::Call>) -> C::Call {
+		UtilityCall::batch_all(calls).into()
+	}
+}
+
+/// Bizinikiwi-based chain that uses `pezpallet-utility`.
+pub trait ChainWithUtilityPallet: Chain {
+	/// The utility pezpallet provider.
+	type UtilityPallet: UtilityPallet<Self>;
+}

pezbridges/relays/client-bizinikiwi/src/client/caching.rs

@@ -0,0 +1,472 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Client implementation that is caching (whenever possible) results of its backend
+//! method calls.
+
+use crate::{
+	client::{Client, SubscriptionBroadcaster},
+	error::{Error, Result},
+	AccountIdOf, AccountKeyPairOf, BlockNumberOf, Chain, ChainWithGrandpa, ChainWithTransactions,
+	HashOf, HeaderIdOf, HeaderOf, NonceOf, SignedBlockOf, SimpleRuntimeVersion, Subscription,
+	TransactionTracker, UnsignedTransaction, ANCIENT_BLOCK_THRESHOLD,
+};
+use std::{cmp::Ordering, future::Future, task::Poll};
+
+use async_std::{
+	sync::{Arc, Mutex, RwLock},
+	task::JoinHandle,
+};
+use async_trait::async_trait;
+use codec::Encode;
+use pezframe_support::weights::Weight;
+use futures::{FutureExt, StreamExt};
+use quick_cache::unsync::Cache;
+use pezsp_consensus_grandpa::{AuthorityId, OpaqueKeyOwnershipProof, SetId};
+use pezsp_core::{
+	storage::{StorageData, StorageKey},
+	Bytes, Pair,
+};
+use pezsp_runtime::{traits::Header as _, transaction_validity::TransactionValidity};
+use pezsp_trie::StorageProof;
+use pezsp_version::RuntimeVersion;
+
+/// `quick_cache::unsync::Cache` wrapped in async-aware synchronization primitives.
+type SyncCache<K, V> = Arc<RwLock<Cache<K, V>>>;
+
+/// Client implementation that is caching (whenever possible) results of its backend
+/// method calls. Apart from caching call results, it also supports some (at the
+/// moment: justifications) subscription sharing, meaning that the single server
+/// subscription may be shared by multiple subscribers at the client side.
+#[derive(Clone)]
+pub struct CachingClient<C: Chain, B: Client<C>> {
+	backend: B,
+	data: Arc<ClientData<C>>,
+}
+
+/// Client data, shared by all `CachingClient` clones.
+struct ClientData<C: Chain> {
+	grandpa_justifications: Arc<Mutex<Option<SubscriptionBroadcaster<Bytes>>>>,
+	beefy_justifications: Arc<Mutex<Option<SubscriptionBroadcaster<Bytes>>>>,
+	background_task_handle: Arc<Mutex<JoinHandle<Result<()>>>>,
+	best_header: Arc<RwLock<Option<HeaderOf<C>>>>,
+	best_finalized_header: Arc<RwLock<Option<HeaderOf<C>>>>,
+	// `quick_cache::sync::Cache` has the `get_or_insert_async` method, which fits our needs,
+	// but it uses synchronization primitives that are not aware of async execution. They
+	// can block the executor threads and cause deadlocks => let's use primitives from
+	// `async_std` crate around `quick_cache::unsync::Cache`
+	header_hash_by_number_cache: SyncCache<BlockNumberOf<C>, HashOf<C>>,
+	header_by_hash_cache: SyncCache<HashOf<C>, HeaderOf<C>>,
+	block_by_hash_cache: SyncCache<HashOf<C>, SignedBlockOf<C>>,
+	raw_storage_value_cache: SyncCache<(HashOf<C>, StorageKey), Option<StorageData>>,
+	state_call_cache: SyncCache<(HashOf<C>, String, Bytes), Bytes>,
+}
+
+impl<C: Chain, B: Client<C>> CachingClient<C, B> {
+	/// Creates new `CachingClient` on top of given `backend`.
+	pub async fn new(backend: B) -> Self {
+		// most of relayer operations will never touch more than `ANCIENT_BLOCK_THRESHOLD`
+		// headers, so we'll use this as a cache capacity for all chain-related caches
+		let chain_state_capacity = ANCIENT_BLOCK_THRESHOLD as usize;
+		let best_header = Arc::new(RwLock::new(None));
+		let best_finalized_header = Arc::new(RwLock::new(None));
+		let header_by_hash_cache = Arc::new(RwLock::new(Cache::new(chain_state_capacity)));
+		let background_task_handle = Self::start_background_task(
+			backend.clone(),
+			best_header.clone(),
+			best_finalized_header.clone(),
+			header_by_hash_cache.clone(),
+		)
+		.await;
+		CachingClient {
+			backend,
+			data: Arc::new(ClientData {
+				grandpa_justifications: Arc::new(Mutex::new(None)),
+				beefy_justifications: Arc::new(Mutex::new(None)),
+				background_task_handle: Arc::new(Mutex::new(background_task_handle)),
+				best_header,
+				best_finalized_header,
+				header_hash_by_number_cache: Arc::new(RwLock::new(Cache::new(
+					chain_state_capacity,
+				))),
+				header_by_hash_cache,
+				block_by_hash_cache: Arc::new(RwLock::new(Cache::new(chain_state_capacity))),
+				raw_storage_value_cache: Arc::new(RwLock::new(Cache::new(1_024))),
+				state_call_cache: Arc::new(RwLock::new(Cache::new(1_024))),
+			}),
+		}
+	}
+
+	/// Try to get value from the cache, or compute and insert it using given future.
+	async fn get_or_insert_async<K: Clone + std::fmt::Debug + Eq + std::hash::Hash, V: Clone>(
+		&self,
+		cache: &Arc<RwLock<Cache<K, V>>>,
+		key: &K,
+		with: impl std::future::Future<Output = Result<V>>,
+	) -> Result<V> {
+		// try to get cached value first using read lock
+		{
+			let cache = cache.read().await;
+			if let Some(value) = cache.get(key) {
+				return Ok(value.clone());
+			}
+		}
+
+		// let's compute the value without holding any locks - it may cause additional misses and
+		// double insertions, but that's better than holding a lock for a while
+		let value = with.await?;
+
+		// insert/update the value in the cache
+		cache.write().await.insert(key.clone(), value.clone());
+		Ok(value)
+	}
+
+	/// Subscribe to finality justifications, trying to reuse existing subscription.
+	async fn subscribe_finality_justifications<'a>(
+		&'a self,
+		maybe_broadcaster: &Mutex<Option<SubscriptionBroadcaster<Bytes>>>,
+		do_subscribe: impl Future<Output = Result<Subscription<Bytes>>> + 'a,
+	) -> Result<Subscription<Bytes>> {
+		let mut maybe_broadcaster = maybe_broadcaster.lock().await;
+		let broadcaster = match maybe_broadcaster.as_ref() {
+			Some(justifications) => justifications,
+			None => {
+				let broadcaster = match SubscriptionBroadcaster::new(do_subscribe.await?) {
+					Ok(broadcaster) => broadcaster,
+					Err(subscription) => return Ok(subscription),
+				};
+				maybe_broadcaster.get_or_insert(broadcaster)
+			},
+		};
+
+		broadcaster.subscribe().await
+	}
+
+	/// Start background task that reads best (and best finalized) headers from subscriptions.
+	async fn start_background_task(
+		backend: B,
+		best_header: Arc<RwLock<Option<HeaderOf<C>>>>,
+		best_finalized_header: Arc<RwLock<Option<HeaderOf<C>>>>,
+		header_by_hash_cache: SyncCache<HashOf<C>, HeaderOf<C>>,
+	) -> JoinHandle<Result<()>> {
+		async_std::task::spawn(async move {
+			// initialize by reading headers directly from backend to avoid doing that in the
+			// high-level code
+			let mut last_finalized_header =
+				backend.header_by_hash(backend.best_finalized_header_hash().await?).await?;
+			*best_header.write().await = Some(backend.best_header().await?);
+			*best_finalized_header.write().await = Some(last_finalized_header.clone());
+
+			// ...and then continue with subscriptions
+			let mut best_headers = backend.subscribe_best_headers().await?;
+			let mut finalized_headers = backend.subscribe_finalized_headers().await?;
+			loop {
+				futures::select! {
+					new_best_header = best_headers.next().fuse() => {
+						// we assume that the best header is always the actual best header, even if its
+						// number is lower than the number of previous-best-header (chain may use its own
+						// best header selection algorithms)
+						let new_best_header = new_best_header
+							.ok_or_else(|| Error::ChannelError(format!("Mandatory best headers subscription for {} has finished", C::NAME)))?;
+						let new_best_header_hash = new_best_header.hash();
+						header_by_hash_cache.write().await.insert(new_best_header_hash, new_best_header.clone());
+						*best_header.write().await = Some(new_best_header);
+					},
+					new_finalized_header = finalized_headers.next().fuse() => {
+						// in theory we'll always get finalized headers in order, but let's double check
+						let new_finalized_header = new_finalized_header.
+							ok_or_else(|| Error::ChannelError(format!("Finalized headers subscription for {} has finished", C::NAME)))?;
+						let new_finalized_header_number = *new_finalized_header.number();
+						let last_finalized_header_number = *last_finalized_header.number();
+						match new_finalized_header_number.cmp(&last_finalized_header_number) {
+							Ordering::Greater => {
+								let new_finalized_header_hash = new_finalized_header.hash();
+								header_by_hash_cache.write().await.insert(new_finalized_header_hash, new_finalized_header.clone());
+								*best_finalized_header.write().await = Some(new_finalized_header.clone());
+								last_finalized_header = new_finalized_header;
+							},
+							Ordering::Less => {
+								return Err(Error::unordered_finalized_headers::<C>(
+									new_finalized_header_number,
+									last_finalized_header_number,
+								));
+							},
+							_ => (),
+						}
+					},
+				}
+			}
+		})
+	}
+
+	/// Ensure that the background task is active.
+	async fn ensure_background_task_active(&self) -> Result<()> {
+		let mut background_task_handle = self.data.background_task_handle.lock().await;
+		if let Poll::Ready(result) = futures::poll!(&mut *background_task_handle) {
+			return Err(Error::ChannelError(format!(
+				"Background task of {} client has exited with result: {:?}",
+				C::NAME,
+				result
+			)));
+		}
+
+		Ok(())
+	}
+
+	/// Try to get header, read elsewhere by background task through subscription.
+	async fn read_header_from_background<'a>(
+		&'a self,
+		header: &Arc<RwLock<Option<HeaderOf<C>>>>,
+		read_header_from_backend: impl Future<Output = Result<HeaderOf<C>>> + 'a,
+	) -> Result<HeaderOf<C>> {
+		// ensure that the background task is active
+		self.ensure_background_task_active().await?;
+
+		// now we know that the background task is active, so we could trust that the
+		// `header` has the most recent updates from it
+		match header.read().await.clone() {
+			Some(header) => Ok(header),
+			None => {
+				// header has not yet been read from the subscription, which means that
+				// we are just starting - let's read header directly from backend this time
+				read_header_from_backend.await
+			},
+		}
+	}
+}
+
+impl<C: Chain, B: Client<C>> std::fmt::Debug for CachingClient<C, B> {
+	fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+		fmt.write_fmt(format_args!("CachingClient<{:?}>", self.backend))
+	}
+}
+
+#[async_trait]
+impl<C: Chain, B: Client<C>> Client<C> for CachingClient<C, B> {
+	async fn ensure_synced(&self) -> Result<()> {
+		self.backend.ensure_synced().await
+	}
+
+	async fn reconnect(&self) -> Result<()> {
+		self.backend.reconnect().await?;
+		// since we have new underlying client, we need to restart subscriptions too
+		*self.data.grandpa_justifications.lock().await = None;
+		*self.data.beefy_justifications.lock().await = None;
+		// also restart background task too
+		*self.data.best_header.write().await = None;
+		*self.data.best_finalized_header.write().await = None;
+		*self.data.background_task_handle.lock().await = Self::start_background_task(
+			self.backend.clone(),
+			self.data.best_header.clone(),
+			self.data.best_finalized_header.clone(),
+			self.data.header_by_hash_cache.clone(),
+		)
+		.await;
+		Ok(())
+	}
+
+	fn genesis_hash(&self) -> HashOf<C> {
+		self.backend.genesis_hash()
+	}
+
+	async fn header_hash_by_number(&self, number: BlockNumberOf<C>) -> Result<HashOf<C>> {
+		self.get_or_insert_async(
+			&self.data.header_hash_by_number_cache,
+			&number,
+			self.backend.header_hash_by_number(number),
+		)
+		.await
+	}
+
+	async fn header_by_hash(&self, hash: HashOf<C>) -> Result<HeaderOf<C>> {
+		self.get_or_insert_async(
+			&self.data.header_by_hash_cache,
+			&hash,
+			self.backend.header_by_hash(hash),
+		)
+		.await
+	}
+
+	async fn block_by_hash(&self, hash: HashOf<C>) -> Result<SignedBlockOf<C>> {
+		self.get_or_insert_async(
+			&self.data.block_by_hash_cache,
+			&hash,
+			self.backend.block_by_hash(hash),
+		)
+		.await
+	}
+
+	async fn best_finalized_header_hash(&self) -> Result<HashOf<C>> {
+		self.read_header_from_background(
+			&self.data.best_finalized_header,
+			self.backend.best_finalized_header(),
+		)
+		.await
+		.map(|h| h.hash())
+	}
+
+	async fn best_header(&self) -> Result<HeaderOf<C>> {
+		self.read_header_from_background(&self.data.best_header, self.backend.best_header())
+			.await
+	}
+
+	async fn subscribe_best_headers(&self) -> Result<Subscription<HeaderOf<C>>> {
+		// we may share the sunbscription here, but atm there's no callers of this method
+		self.backend.subscribe_best_headers().await
+	}
+
+	async fn subscribe_finalized_headers(&self) -> Result<Subscription<HeaderOf<C>>> {
+		// we may share the sunbscription here, but atm there's no callers of this method
+		self.backend.subscribe_finalized_headers().await
+	}
+
+	async fn subscribe_grandpa_finality_justifications(&self) -> Result<Subscription<Bytes>>
+	where
+		C: ChainWithGrandpa,
+	{
+		self.subscribe_finality_justifications(
+			&self.data.grandpa_justifications,
+			self.backend.subscribe_grandpa_finality_justifications(),
+		)
+		.await
+	}
+
+	async fn generate_grandpa_key_ownership_proof(
+		&self,
+		at: HashOf<C>,
+		set_id: SetId,
+		authority_id: AuthorityId,
+	) -> Result<Option<OpaqueKeyOwnershipProof>> {
+		self.backend
+			.generate_grandpa_key_ownership_proof(at, set_id, authority_id)
+			.await
+	}
+
+	async fn subscribe_beefy_finality_justifications(&self) -> Result<Subscription<Bytes>> {
+		self.subscribe_finality_justifications(
+			&self.data.beefy_justifications,
+			self.backend.subscribe_beefy_finality_justifications(),
+		)
+		.await
+	}
+
+	async fn token_decimals(&self) -> Result<Option<u64>> {
+		self.backend.token_decimals().await
+	}
+
+	async fn runtime_version(&self) -> Result<RuntimeVersion> {
+		self.backend.runtime_version().await
+	}
+
+	async fn simple_runtime_version(&self) -> Result<SimpleRuntimeVersion> {
+		self.backend.simple_runtime_version().await
+	}
+
+	fn can_start_version_guard(&self) -> bool {
+		self.backend.can_start_version_guard()
+	}
+
+	async fn raw_storage_value(
+		&self,
+		at: HashOf<C>,
+		storage_key: StorageKey,
+	) -> Result<Option<StorageData>> {
+		self.get_or_insert_async(
+			&self.data.raw_storage_value_cache,
+			&(at, storage_key.clone()),
+			self.backend.raw_storage_value(at, storage_key),
+		)
+		.await
+	}
+
+	async fn pending_extrinsics(&self) -> Result<Vec<Bytes>> {
+		self.backend.pending_extrinsics().await
+	}
+
+	async fn submit_unsigned_extrinsic(&self, transaction: Bytes) -> Result<HashOf<C>> {
+		self.backend.submit_unsigned_extrinsic(transaction).await
+	}
+
+	async fn submit_signed_extrinsic(
+		&self,
+		signer: &AccountKeyPairOf<C>,
+		prepare_extrinsic: impl FnOnce(HeaderIdOf<C>, NonceOf<C>) -> Result<UnsignedTransaction<C>>
+			+ Send
+			+ 'static,
+	) -> Result<HashOf<C>>
+	where
+		C: ChainWithTransactions,
+		AccountIdOf<C>: From<<AccountKeyPairOf<C> as Pair>::Public>,
+	{
+		self.backend.submit_signed_extrinsic(signer, prepare_extrinsic).await
+	}
+
+	async fn submit_and_watch_signed_extrinsic(
+		&self,
+		signer: &AccountKeyPairOf<C>,
+		prepare_extrinsic: impl FnOnce(HeaderIdOf<C>, NonceOf<C>) -> Result<UnsignedTransaction<C>>
+			+ Send
+			+ 'static,
+	) -> Result<TransactionTracker<C, Self>>
+	where
+		C: ChainWithTransactions,
+		AccountIdOf<C>: From<<AccountKeyPairOf<C> as Pair>::Public>,
+	{
+		self.backend
+			.submit_and_watch_signed_extrinsic(signer, prepare_extrinsic)
+			.await
+			.map(|t| t.switch_environment(self.clone()))
+	}
+
+	async fn validate_transaction<SignedTransaction: Encode + Send + 'static>(
+		&self,
+		at: HashOf<C>,
+		transaction: SignedTransaction,
+	) -> Result<TransactionValidity> {
+		self.backend.validate_transaction(at, transaction).await
+	}
+
+	async fn estimate_extrinsic_weight<SignedTransaction: Encode + Send + 'static>(
+		&self,
+		at: HashOf<C>,
+		transaction: SignedTransaction,
+	) -> Result<Weight> {
+		self.backend.estimate_extrinsic_weight(at, transaction).await
+	}
+
+	async fn raw_state_call<Args: Encode + Send>(
+		&self,
+		at: HashOf<C>,
+		method: String,
+		arguments: Args,
+	) -> Result<Bytes> {
+		let encoded_arguments = Bytes(arguments.encode());
+		self.get_or_insert_async(
+			&self.data.state_call_cache,
+			&(at, method.clone(), encoded_arguments),
+			self.backend.raw_state_call(at, method, arguments),
+		)
+		.await
+	}
+
+	async fn prove_storage(
+		&self,
+		at: HashOf<C>,
+		keys: Vec<StorageKey>,
+	) -> Result<(StorageProof, HashOf<C>)> {
+		self.backend.prove_storage(at, keys).await
+	}
+}

pezbridges/relays/client-bizinikiwi/src/client/mod.rs

@@ -0,0 +1,91 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Layered Bizinikiwi client implementation.
+
+use crate::{Chain, ConnectionParams};
+
+use caching::CachingClient;
+use num_traits::Saturating;
+use rpc::RpcClient;
+use pezsp_version::RuntimeVersion;
+
+pub mod caching;
+pub mod rpc;
+
+mod rpc_api;
+mod subscription;
+mod traits;
+
+pub use subscription::{StreamDescription, Subscription, SubscriptionBroadcaster};
+pub use traits::Client;
+
+/// Type of RPC client with caching support.
+pub type RpcWithCachingClient<C> = CachingClient<C, RpcClient<C>>;
+
+/// Creates new RPC client with caching support.
+pub async fn rpc_with_caching<C: Chain>(params: ConnectionParams) -> RpcWithCachingClient<C> {
+	let rpc = rpc::RpcClient::<C>::new(params).await;
+	caching::CachingClient::new(rpc).await
+}
+
+/// The difference between best block number and number of its ancestor, that is enough
+/// for us to consider that ancestor an "ancient" block with dropped state.
+///
+/// The relay does not assume that it is connected to the archive node, so it always tries
+/// to use the best available chain state. But sometimes it still may use state of some
+/// old block. If the state of that block is already dropped, relay will see errors when
+/// e.g. it tries to prove something.
+///
+/// By default Bizinikiwi-based nodes are storing state for last 256 blocks. We'll use
+/// half of this value.
+pub const ANCIENT_BLOCK_THRESHOLD: u32 = 128;
+
+/// Returns `true` if we think that the state is already discarded for given block.
+pub fn is_ancient_block<N: From<u32> + PartialOrd + Saturating>(block: N, best: N) -> bool {
+	best.saturating_sub(block) >= N::from(ANCIENT_BLOCK_THRESHOLD)
+}
+
+/// Opaque GRANDPA authorities set.
+pub type OpaqueGrandpaAuthoritiesSet = Vec<u8>;
+
+/// A simple runtime version. It only includes the `spec_version` and `transaction_version`.
+#[derive(Copy, Clone, Debug)]
+pub struct SimpleRuntimeVersion {
+	/// Version of the runtime specification.
+	pub spec_version: u32,
+	/// All existing dispatches are fully compatible when this number doesn't change.
+	pub transaction_version: u32,
+}
+
+impl SimpleRuntimeVersion {
+	/// Create a new instance of `SimpleRuntimeVersion` from a `RuntimeVersion`.
+	pub const fn from_runtime_version(runtime_version: &RuntimeVersion) -> Self {
+		Self {
+			spec_version: runtime_version.spec_version,
+			transaction_version: runtime_version.transaction_version,
+		}
+	}
+}
+
+/// Chain runtime version in client
+#[derive(Copy, Clone, Debug)]
+pub enum ChainRuntimeVersion {
+	/// Auto query from chain.
+	Auto,
+	/// Custom runtime version, defined by user.
+	Custom(SimpleRuntimeVersion),
+}

pezbridges/relays/client-bizinikiwi/src/client/rpc.rs

@@ -0,0 +1,743 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Client implementation that connects to the Bizinikiwi node over `ws`/`wss` connection
+//! and is using RPC methods to get required data and submit transactions.
+
+use crate::{
+	client::{
+		rpc_api::{
+			BizinikiwiAuthorClient, BizinikiwiBeefyClient, BizinikiwiChainClient,
+			BizinikiwiFrameSystemClient, BizinikiwiGrandpaClient, BizinikiwiStateClient,
+			BizinikiwiSystemClient,
+		},
+		subscription::{StreamDescription, Subscription},
+		Client,
+	},
+	error::{Error, Result},
+	guard::Environment,
+	transaction_stall_timeout, AccountIdOf, AccountKeyPairOf, BalanceOf, BlockNumberOf, Chain,
+	ChainRuntimeVersion, ChainWithGrandpa, ChainWithTransactions, ConnectionParams, HashOf,
+	HeaderIdOf, HeaderOf, NonceOf, SignParam, SignedBlockOf, SimpleRuntimeVersion,
+	TransactionTracker, UnsignedTransaction,
+};
+
+use async_std::sync::{Arc, Mutex, RwLock};
+use async_trait::async_trait;
+use pezbp_runtime::HeaderIdProvider;
+use codec::Encode;
+use pezframe_support::weights::Weight;
+use futures::TryFutureExt;
+use jsonrpsee::{
+	core::{client::Subscription as RpcSubscription, ClientError},
+	ws_client::{WsClient, WsClientBuilder},
+};
+use num_traits::Zero;
+use pezpallet_transaction_payment::RuntimeDispatchInfo;
+use relay_utils::{relay_loop::RECONNECT_DELAY, STALL_TIMEOUT};
+use pezsp_core::{
+	storage::{StorageData, StorageKey},
+	Bytes, Hasher, Pair,
+};
+use pezsp_runtime::{
+	traits::Header,
+	transaction_validity::{TransactionSource, TransactionValidity},
+};
+use pezsp_trie::StorageProof;
+use pezsp_version::RuntimeVersion;
+use std::{cmp::Ordering, future::Future, marker::PhantomData};
+
+const MAX_SUBSCRIPTION_CAPACITY: usize = 4096;
+
+const SUB_API_TXPOOL_VALIDATE_TRANSACTION: &str = "TaggedTransactionQueue_validate_transaction";
+const SUB_API_TX_PAYMENT_QUERY_INFO: &str = "TransactionPaymentApi_query_info";
+const SUB_API_GRANDPA_GENERATE_KEY_OWNERSHIP_PROOF: &str =
+	"GrandpaApi_generate_key_ownership_proof";
+
+/// Client implementation that connects to the Bizinikiwi node over `ws`/`wss` connection
+/// and is using RPC methods to get required data and submit transactions.
+pub struct RpcClient<C: Chain> {
+	// Lock order: `submit_signed_extrinsic_lock`, `data`
+	/// Client connection params.
+	params: Arc<ConnectionParams>,
+	/// If several tasks are submitting their transactions simultaneously using
+	/// `submit_signed_extrinsic` method, they may get the same transaction nonce. So one of
+	/// transactions will be rejected from the pool. This lock is here to prevent situations like
+	/// that.
+	submit_signed_extrinsic_lock: Arc<Mutex<()>>,
+	/// Genesis block hash.
+	genesis_hash: HashOf<C>,
+	/// Shared dynamic data.
+	data: Arc<RwLock<ClientData>>,
+	/// Generic arguments dump.
+	_phantom: PhantomData<C>,
+}
+
+/// Client data, shared by all `RpcClient` clones.
+struct ClientData {
+	/// Tokio runtime handle.
+	tokio: Arc<tokio::runtime::Runtime>,
+	/// Bizinikiwi RPC client.
+	client: Arc<WsClient>,
+}
+
+/// Already encoded value.
+struct PreEncoded(Vec<u8>);
+
+impl Encode for PreEncoded {
+	fn encode(&self) -> Vec<u8> {
+		self.0.clone()
+	}
+}
+
+impl<C: Chain> std::fmt::Debug for RpcClient<C> {
+	fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+		fmt.write_fmt(format_args!("RpcClient<{}>", C::NAME))
+	}
+}
+
+impl<C: Chain> RpcClient<C> {
+	/// Returns client that is able to call RPCs on Bizinikiwi node over websocket connection.
+	///
+	/// This function will keep connecting to given Bizinikiwi node until connection is established
+	/// and is functional. If attempt fail, it will wait for `RECONNECT_DELAY` and retry again.
+	pub async fn new(params: ConnectionParams) -> Self {
+		let params = Arc::new(params);
+		loop {
+			match Self::try_connect(params.clone()).await {
+				Ok(client) => return client,
+				Err(error) => tracing::error!(
+					target: "bridge",
+					?error,
+					node=%C::NAME,
+					retry_as_secs=%RECONNECT_DELAY.as_secs(),
+					"Failed to connect. Going to retry"
+				),
+			}
+
+			async_std::task::sleep(RECONNECT_DELAY).await;
+		}
+	}
+
+	/// Try to connect to Bizinikiwi node over websocket. Returns Bizinikiwi RPC client if connection
+	/// has been established or error otherwise.
+	async fn try_connect(params: Arc<ConnectionParams>) -> Result<Self> {
+		let (tokio, client) = Self::build_client(&params).await?;
+
+		let genesis_hash_client = client.clone();
+		let genesis_hash = tokio
+			.spawn(async move {
+				BizinikiwiChainClient::<C>::block_hash(&*genesis_hash_client, Some(Zero::zero()))
+					.await
+			})
+			.await??;
+
+		let chain_runtime_version = params.chain_runtime_version;
+		let mut client = Self {
+			params,
+			submit_signed_extrinsic_lock: Arc::new(Mutex::new(())),
+			genesis_hash,
+			data: Arc::new(RwLock::new(ClientData { tokio, client })),
+			_phantom: PhantomData,
+		};
+		Self::ensure_correct_runtime_version(&mut client, chain_runtime_version).await?;
+		Ok(client)
+	}
+
+	// Check runtime version to understand if we need are connected to expected version, or we
+	// need to wait for upgrade, we need to abort immediately.
+	async fn ensure_correct_runtime_version<E: Environment<C, Error = Error>>(
+		env: &mut E,
+		expected: ChainRuntimeVersion,
+	) -> Result<()> {
+		// we are only interested if version mode is bundled or passed using CLI
+		let expected = match expected {
+			ChainRuntimeVersion::Auto => return Ok(()),
+			ChainRuntimeVersion::Custom(expected) => expected,
+		};
+
+		// we need to wait if actual version is < than expected, we are OK of versions are the
+		// same and we need to abort if actual version is > than expected
+		let actual = SimpleRuntimeVersion::from_runtime_version(&env.runtime_version().await?);
+		match actual.spec_version.cmp(&expected.spec_version) {
+			Ordering::Less =>
+				Err(Error::WaitingForRuntimeUpgrade { chain: C::NAME.into(), expected, actual }),
+			Ordering::Equal => Ok(()),
+			Ordering::Greater => {
+				tracing::error!(
+					target: "bridge",
+					node=%C::NAME,
+					?expected,
+					?actual,
+					"The client is configured to use runtime version, which is different from the \
+					actual version. Aborting",
+				);
+				env.abort().await;
+				Err(Error::Custom("Aborted".into()))
+			},
+		}
+	}
+
+	/// Build client to use in connection.
+	async fn build_client(
+		params: &ConnectionParams,
+	) -> Result<(Arc<tokio::runtime::Runtime>, Arc<WsClient>)> {
+		let tokio = tokio::runtime::Runtime::new()?;
+		let uri = params.uri.clone();
+		tracing::info!(target: "bridge", node=%C::NAME, %uri, "Connecting");
+
+		let client = tokio
+			.spawn(async move {
+				WsClientBuilder::default()
+					.max_buffer_capacity_per_subscription(MAX_SUBSCRIPTION_CAPACITY)
+					.build(&uri)
+					.await
+			})
+			.await??;
+
+		Ok((Arc::new(tokio), Arc::new(client)))
+	}
+
+	/// Execute jsonrpsee future in tokio context.
+	async fn jsonrpsee_execute<MF, F, T>(&self, make_jsonrpsee_future: MF) -> Result<T>
+	where
+		MF: FnOnce(Arc<WsClient>) -> F + Send + 'static,
+		F: Future<Output = Result<T>> + Send + 'static,
+		T: Send + 'static,
+	{
+		let data = self.data.read().await;
+		let client = data.client.clone();
+		data.tokio.spawn(make_jsonrpsee_future(client)).await?
+	}
+
+	/// Prepare parameters used to sign chain transactions.
+	async fn build_sign_params(&self, signer: AccountKeyPairOf<C>) -> Result<SignParam<C>>
+	where
+		C: ChainWithTransactions,
+	{
+		let runtime_version = self.simple_runtime_version().await?;
+		Ok(SignParam::<C> {
+			spec_version: runtime_version.spec_version,
+			transaction_version: runtime_version.transaction_version,
+			genesis_hash: self.genesis_hash,
+			signer,
+		})
+	}
+
+	/// Get the nonce of the given Bizinikiwi account.
+	pub async fn next_account_index(&self, account: AccountIdOf<C>) -> Result<NonceOf<C>> {
+		self.jsonrpsee_execute(move |client| async move {
+			Ok(BizinikiwiFrameSystemClient::<C>::account_next_index(&*client, account).await?)
+		})
+		.await
+	}
+
+	/// Subscribe to finality justifications.
+	async fn subscribe_finality_justifications<Fut>(
+		&self,
+		gadget_name: &str,
+		do_subscribe: impl FnOnce(Arc<WsClient>) -> Fut + Send + 'static,
+	) -> Result<Subscription<Bytes>>
+	where
+		Fut: Future<Output = std::result::Result<RpcSubscription<Bytes>, ClientError>> + Send,
+	{
+		let subscription = self
+			.jsonrpsee_execute(move |client| async move { Ok(do_subscribe(client).await?) })
+			.map_err(|e| Error::failed_to_subscribe_justification::<C>(e))
+			.await?;
+
+		Ok(Subscription::new_forwarded(
+			StreamDescription::new(format!("{} justifications", gadget_name), C::NAME.into()),
+			subscription,
+		))
+	}
+
+	/// Subscribe to headers stream.
+	async fn subscribe_headers<Fut>(
+		&self,
+		stream_name: &str,
+		do_subscribe: impl FnOnce(Arc<WsClient>) -> Fut + Send + 'static,
+		map_err: impl FnOnce(Error) -> Error,
+	) -> Result<Subscription<HeaderOf<C>>>
+	where
+		Fut: Future<Output = std::result::Result<RpcSubscription<HeaderOf<C>>, ClientError>> + Send,
+	{
+		let subscription = self
+			.jsonrpsee_execute(move |client| async move { Ok(do_subscribe(client).await?) })
+			.map_err(map_err)
+			.await?;
+
+		Ok(Subscription::new_forwarded(
+			StreamDescription::new(format!("{} headers", stream_name), C::NAME.into()),
+			subscription,
+		))
+	}
+}
+
+impl<C: Chain> Clone for RpcClient<C> {
+	fn clone(&self) -> Self {
+		RpcClient {
+			params: self.params.clone(),
+			submit_signed_extrinsic_lock: self.submit_signed_extrinsic_lock.clone(),
+			genesis_hash: self.genesis_hash,
+			data: self.data.clone(),
+			_phantom: PhantomData,
+		}
+	}
+}
+
+#[async_trait]
+impl<C: Chain> Client<C> for RpcClient<C> {
+	async fn ensure_synced(&self) -> Result<()> {
+		let health = self
+			.jsonrpsee_execute(|client| async move {
+				Ok(BizinikiwiSystemClient::<C>::health(&*client).await?)
+			})
+			.await
+			.map_err(|e| Error::failed_to_get_system_health::<C>(e))?;
+
+		let is_synced = !health.is_syncing && (!health.should_have_peers || health.peers > 0);
+		if is_synced {
+			Ok(())
+		} else {
+			Err(Error::ClientNotSynced(health))
+		}
+	}
+
+	async fn reconnect(&self) -> Result<()> {
+		let mut data = self.data.write().await;
+		let (tokio, client) = Self::build_client(&self.params).await?;
+		data.tokio = tokio;
+		data.client = client;
+		Ok(())
+	}
+
+	fn genesis_hash(&self) -> HashOf<C> {
+		self.genesis_hash
+	}
+
+	async fn header_hash_by_number(&self, number: BlockNumberOf<C>) -> Result<HashOf<C>> {
+		self.jsonrpsee_execute(move |client| async move {
+			Ok(BizinikiwiChainClient::<C>::block_hash(&*client, Some(number)).await?)
+		})
+		.await
+		.map_err(|e| Error::failed_to_read_header_hash_by_number::<C>(number, e))
+	}
+
+	async fn header_by_hash(&self, hash: HashOf<C>) -> Result<HeaderOf<C>> {
+		self.jsonrpsee_execute(move |client| async move {
+			Ok(BizinikiwiChainClient::<C>::header(&*client, Some(hash)).await?)
+		})
+		.await
+		.map_err(|e| Error::failed_to_read_header_by_hash::<C>(hash, e))
+	}
+
+	async fn block_by_hash(&self, hash: HashOf<C>) -> Result<SignedBlockOf<C>> {
+		self.jsonrpsee_execute(move |client| async move {
+			Ok(BizinikiwiChainClient::<C>::block(&*client, Some(hash)).await?)
+		})
+		.await
+		.map_err(|e| Error::failed_to_read_block_by_hash::<C>(hash, e))
+	}
+
+	async fn best_finalized_header_hash(&self) -> Result<HashOf<C>> {
+		self.jsonrpsee_execute(|client| async move {
+			Ok(BizinikiwiChainClient::<C>::finalized_head(&*client).await?)
+		})
+		.await
+		.map_err(|e| Error::failed_to_read_best_finalized_header_hash::<C>(e))
+	}
+
+	async fn best_header(&self) -> Result<HeaderOf<C>> {
+		self.jsonrpsee_execute(|client| async move {
+			Ok(BizinikiwiChainClient::<C>::header(&*client, None).await?)
+		})
+		.await
+		.map_err(|e| Error::failed_to_read_best_header::<C>(e))
+	}
+
+	async fn subscribe_best_headers(&self) -> Result<Subscription<HeaderOf<C>>> {
+		self.subscribe_headers(
+			"best headers",
+			move |client| async move { BizinikiwiChainClient::<C>::subscribe_new_heads(&*client).await },
+			|e| Error::failed_to_subscribe_best_headers::<C>(e),
+		)
+		.await
+	}
+
+	async fn subscribe_finalized_headers(&self) -> Result<Subscription<HeaderOf<C>>> {
+		self.subscribe_headers(
+			"best finalized headers",
+			move |client| async move {
+				BizinikiwiChainClient::<C>::subscribe_finalized_heads(&*client).await
+			},
+			|e| Error::failed_to_subscribe_finalized_headers::<C>(e),
+		)
+		.await
+	}
+
+	async fn subscribe_grandpa_finality_justifications(&self) -> Result<Subscription<Bytes>>
+	where
+		C: ChainWithGrandpa,
+	{
+		self.subscribe_finality_justifications("GRANDPA", move |client| async move {
+			BizinikiwiGrandpaClient::<C>::subscribe_justifications(&*client).await
+		})
+		.await
+	}
+
+	async fn generate_grandpa_key_ownership_proof(
+		&self,
+		at: HashOf<C>,
+		set_id: pezsp_consensus_grandpa::SetId,
+		authority_id: pezsp_consensus_grandpa::AuthorityId,
+	) -> Result<Option<pezsp_consensus_grandpa::OpaqueKeyOwnershipProof>> {
+		self.state_call(
+			at,
+			SUB_API_GRANDPA_GENERATE_KEY_OWNERSHIP_PROOF.into(),
+			(set_id, authority_id),
+		)
+		.await
+	}
+
+	async fn subscribe_beefy_finality_justifications(&self) -> Result<Subscription<Bytes>> {
+		self.subscribe_finality_justifications("BEEFY", move |client| async move {
+			BizinikiwiBeefyClient::<C>::subscribe_justifications(&*client).await
+		})
+		.await
+	}
+
+	async fn token_decimals(&self) -> Result<Option<u64>> {
+		self.jsonrpsee_execute(move |client| async move {
+			let system_properties = BizinikiwiSystemClient::<C>::properties(&*client).await?;
+			Ok(system_properties.get("tokenDecimals").and_then(|v| v.as_u64()))
+		})
+		.await
+	}
+
+	async fn runtime_version(&self) -> Result<RuntimeVersion> {
+		self.jsonrpsee_execute(move |client| async move {
+			Ok(BizinikiwiStateClient::<C>::runtime_version(&*client).await?)
+		})
+		.await
+		.map_err(|e| Error::failed_to_read_runtime_version::<C>(e))
+	}
+
+	async fn simple_runtime_version(&self) -> Result<SimpleRuntimeVersion> {
+		Ok(match self.params.chain_runtime_version {
+			ChainRuntimeVersion::Auto => {
+				let runtime_version = self.runtime_version().await?;
+				SimpleRuntimeVersion::from_runtime_version(&runtime_version)
+			},
+			ChainRuntimeVersion::Custom(ref version) => *version,
+		})
+	}
+
+	fn can_start_version_guard(&self) -> bool {
+		!matches!(self.params.chain_runtime_version, ChainRuntimeVersion::Auto)
+	}
+
+	async fn raw_storage_value(
+		&self,
+		at: HashOf<C>,
+		storage_key: StorageKey,
+	) -> Result<Option<StorageData>> {
+		let cloned_storage_key = storage_key.clone();
+		self.jsonrpsee_execute(move |client| async move {
+			Ok(BizinikiwiStateClient::<C>::storage(&*client, cloned_storage_key, Some(at)).await?)
+		})
+		.await
+		.map_err(|e| Error::failed_to_read_storage_value::<C>(at, storage_key, e))
+	}
+
+	async fn pending_extrinsics(&self) -> Result<Vec<Bytes>> {
+		self.jsonrpsee_execute(move |client| async move {
+			Ok(BizinikiwiAuthorClient::<C>::pending_extrinsics(&*client).await?)
+		})
+		.await
+		.map_err(|e| Error::failed_to_get_pending_extrinsics::<C>(e))
+	}
+
+	async fn submit_unsigned_extrinsic(&self, transaction: Bytes) -> Result<HashOf<C>> {
+		// one last check that the transaction is valid. Most of checks happen in the relay loop and
+		// it is the "final" check before submission.
+		let best_header_hash = self.best_header_hash().await?;
+		self.validate_transaction(best_header_hash, PreEncoded(transaction.0.clone()))
+			.await
+			.map_err(|e| Error::failed_to_submit_transaction::<C>(e))?
+			.map_err(|e| Error::failed_to_submit_transaction::<C>(Error::TransactionInvalid(e)))?;
+
+		self.jsonrpsee_execute(move |client| async move {
+			let tx_hash = BizinikiwiAuthorClient::<C>::submit_extrinsic(&*client, transaction)
+				.await
+				.map_err(|e| {
+					tracing::error!(target: "bridge", error=?e, node=%C::NAME, "Failed to send transaction");
+					e
+				})?;
+			tracing::trace!(target: "bridge", node=%C::NAME, ?tx_hash, "Sent transaction");
+			Ok(tx_hash)
+		})
+		.await
+		.map_err(|e| Error::failed_to_submit_transaction::<C>(e))
+	}
+
+	async fn submit_signed_extrinsic(
+		&self,
+		signer: &AccountKeyPairOf<C>,
+		prepare_extrinsic: impl FnOnce(HeaderIdOf<C>, NonceOf<C>) -> Result<UnsignedTransaction<C>>
+			+ Send
+			+ 'static,
+	) -> Result<HashOf<C>>
+	where
+		C: ChainWithTransactions,
+		AccountIdOf<C>: From<<AccountKeyPairOf<C> as Pair>::Public>,
+	{
+		let _guard = self.submit_signed_extrinsic_lock.lock().await;
+		let transaction_nonce = self.next_account_index(signer.public().into()).await?;
+		let best_header = self.best_header().await?;
+		let signing_data = self.build_sign_params(signer.clone()).await?;
+
+		// By using parent of best block here, we are protecting again best-block reorganizations.
+		// E.g. transaction may have been submitted when the best block was `A[num=100]`. Then it
+		// has been changed to `B[num=100]`. Hash of `A` has been included into transaction
+		// signature payload. So when signature will be checked, the check will fail and transaction
+		// will be dropped from the pool.
+		let best_header_id = best_header.parent_id().unwrap_or_else(|| best_header.id());
+
+		let extrinsic = prepare_extrinsic(best_header_id, transaction_nonce)?;
+		let signed_extrinsic = C::sign_transaction(signing_data, extrinsic)?.encode();
+		self.submit_unsigned_extrinsic(Bytes(signed_extrinsic)).await
+	}
+
+	async fn submit_and_watch_signed_extrinsic(
+		&self,
+		signer: &AccountKeyPairOf<C>,
+		prepare_extrinsic: impl FnOnce(HeaderIdOf<C>, NonceOf<C>) -> Result<UnsignedTransaction<C>>
+			+ Send
+			+ 'static,
+	) -> Result<TransactionTracker<C, Self>>
+	where
+		C: ChainWithTransactions,
+		AccountIdOf<C>: From<<AccountKeyPairOf<C> as Pair>::Public>,
+	{
+		let self_clone = self.clone();
+		let signing_data = self.build_sign_params(signer.clone()).await?;
+		let _guard = self.submit_signed_extrinsic_lock.lock().await;
+		let transaction_nonce = self.next_account_index(signer.public().into()).await?;
+		let best_header = self.best_header().await?;
+		let best_header_id = best_header.id();
+
+		let extrinsic = prepare_extrinsic(best_header_id, transaction_nonce)?;
+		let stall_timeout = transaction_stall_timeout(
+			extrinsic.era.mortality_period(),
+			C::AVERAGE_BLOCK_INTERVAL,
+			STALL_TIMEOUT,
+		);
+		let signed_extrinsic = C::sign_transaction(signing_data, extrinsic)?.encode();
+
+		// one last check that the transaction is valid. Most of checks happen in the relay loop and
+		// it is the "final" check before submission.
+		self.validate_transaction(best_header_id.hash(), PreEncoded(signed_extrinsic.clone()))
+			.await
+			.map_err(|e| Error::failed_to_submit_transaction::<C>(e))?
+			.map_err(|e| Error::failed_to_submit_transaction::<C>(Error::TransactionInvalid(e)))?;
+
+		self.jsonrpsee_execute(move |client| async move {
+			let tx_hash = C::Hasher::hash(&signed_extrinsic);
+			let subscription: jsonrpsee::core::client::Subscription<_> =
+				BizinikiwiAuthorClient::<C>::submit_and_watch_extrinsic(
+					&*client,
+					Bytes(signed_extrinsic),
+				)
+				.await
+				.map_err(|e| {
+					tracing::error!(target: "bridge", error=?e, node=%C::NAME, "Failed to send transaction");
+					e
+				})?;
+			tracing::trace!(target: "bridge", node=%C::NAME, ?tx_hash, "Sent transaction");
+			Ok(TransactionTracker::new(
+				self_clone,
+				stall_timeout,
+				tx_hash,
+				Subscription::new_forwarded(
+					StreamDescription::new("transaction events".into(), C::NAME.into()),
+					subscription,
+				),
+			))
+		})
+		.await
+		.map_err(|e| Error::failed_to_submit_transaction::<C>(e))
+	}
+
+	async fn validate_transaction<SignedTransaction: Encode + Send + 'static>(
+		&self,
+		at: HashOf<C>,
+		transaction: SignedTransaction,
+	) -> Result<TransactionValidity> {
+		self.state_call(
+			at,
+			SUB_API_TXPOOL_VALIDATE_TRANSACTION.into(),
+			(TransactionSource::External, transaction, at),
+		)
+		.await
+	}
+
+	async fn estimate_extrinsic_weight<SignedTransaction: Encode + Send + 'static>(
+		&self,
+		at: HashOf<C>,
+		transaction: SignedTransaction,
+	) -> Result<Weight> {
+		let transaction_len = transaction.encoded_size() as u32;
+		let dispatch_info: RuntimeDispatchInfo<BalanceOf<C>> = self
+			.state_call(at, SUB_API_TX_PAYMENT_QUERY_INFO.into(), (transaction, transaction_len))
+			.await?;
+
+		Ok(dispatch_info.weight)
+	}
+
+	async fn raw_state_call<Args: Encode + Send>(
+		&self,
+		at: HashOf<C>,
+		method: String,
+		arguments: Args,
+	) -> Result<Bytes> {
+		let arguments = Bytes(arguments.encode());
+		let arguments_clone = arguments.clone();
+		let method_clone = method.clone();
+		self.jsonrpsee_execute(move |client| async move {
+			BizinikiwiStateClient::<C>::call(&*client, method, arguments, Some(at))
+				.await
+				.map_err(Into::into)
+		})
+		.await
+		.map_err(|e| Error::failed_state_call::<C>(at, method_clone, arguments_clone, e))
+	}
+
+	async fn prove_storage(
+		&self,
+		at: HashOf<C>,
+		keys: Vec<StorageKey>,
+	) -> Result<(StorageProof, HashOf<C>)> {
+		let state_root = *self.header_by_hash(at).await?.state_root();
+
+		let keys_clone = keys.clone();
+		let read_proof = self
+			.jsonrpsee_execute(move |client| async move {
+				BizinikiwiStateClient::<C>::prove_storage(&*client, keys_clone, Some(at))
+					.await
+					.map(|proof| StorageProof::new(proof.proof.into_iter().map(|b| b.0)))
+					.map_err(Into::into)
+			})
+			.await
+			.map_err(|e| Error::failed_to_prove_storage::<C>(at, keys.clone(), e))?;
+
+		Ok((read_proof, state_root))
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::{guard::tests::TestEnvironment, test_chain::TestChain};
+	use futures::{channel::mpsc::unbounded, FutureExt, SinkExt, StreamExt};
+
+	async fn run_ensure_correct_runtime_version(
+		expected: ChainRuntimeVersion,
+		actual: RuntimeVersion,
+	) -> Result<()> {
+		let (
+			(mut runtime_version_tx, runtime_version_rx),
+			(slept_tx, _slept_rx),
+			(aborted_tx, mut aborted_rx),
+		) = (unbounded(), unbounded(), unbounded());
+		runtime_version_tx.send(actual).await.unwrap();
+		let mut env = TestEnvironment { runtime_version_rx, slept_tx, aborted_tx };
+
+		let ensure_correct_runtime_version =
+			RpcClient::<TestChain>::ensure_correct_runtime_version(&mut env, expected).boxed();
+		let aborted = aborted_rx.next().map(|_| Err(Error::Custom("".into()))).boxed();
+		futures::pin_mut!(ensure_correct_runtime_version, aborted);
+		futures::future::select(ensure_correct_runtime_version, aborted)
+			.await
+			.into_inner()
+			.0
+	}
+
+	#[async_std::test]
+	async fn ensure_correct_runtime_version_works() {
+		// when we are configured to use auto version
+		assert!(matches!(
+			run_ensure_correct_runtime_version(
+				ChainRuntimeVersion::Auto,
+				RuntimeVersion {
+					spec_version: 100,
+					transaction_version: 100,
+					..Default::default()
+				},
+			)
+			.await,
+			Ok(()),
+		));
+		// when actual == expected
+		assert!(matches!(
+			run_ensure_correct_runtime_version(
+				ChainRuntimeVersion::Custom(SimpleRuntimeVersion {
+					spec_version: 100,
+					transaction_version: 100
+				}),
+				RuntimeVersion {
+					spec_version: 100,
+					transaction_version: 100,
+					..Default::default()
+				},
+			)
+			.await,
+			Ok(()),
+		));
+		// when actual spec version < expected spec version
+		assert!(matches!(
+			run_ensure_correct_runtime_version(
+				ChainRuntimeVersion::Custom(SimpleRuntimeVersion {
+					spec_version: 100,
+					transaction_version: 100
+				}),
+				RuntimeVersion { spec_version: 99, transaction_version: 100, ..Default::default() },
+			)
+			.await,
+			Err(Error::WaitingForRuntimeUpgrade {
+				expected: SimpleRuntimeVersion { spec_version: 100, transaction_version: 100 },
+				actual: SimpleRuntimeVersion { spec_version: 99, transaction_version: 100 },
+				..
+			}),
+		));
+		// when actual spec version > expected spec version
+		assert!(matches!(
+			run_ensure_correct_runtime_version(
+				ChainRuntimeVersion::Custom(SimpleRuntimeVersion {
+					spec_version: 100,
+					transaction_version: 100
+				}),
+				RuntimeVersion {
+					spec_version: 101,
+					transaction_version: 100,
+					..Default::default()
+				},
+			)
+			.await,
+			Err(Error::Custom(_)),
+		));
+	}
+}

pezbridges/relays/client-bizinikiwi/src/client/rpc_api.rs

@@ -0,0 +1,152 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The most generic Bizinikiwi node RPC interface.
+
+use crate::{Chain, ChainWithGrandpa, TransactionStatusOf};
+
+use jsonrpsee::proc_macros::rpc;
+use pezpallet_transaction_payment_rpc_runtime_api::FeeDetails;
+use pezsc_rpc_api::{state::ReadProof, system::Health};
+use pezsp_core::{
+	storage::{StorageData, StorageKey},
+	Bytes,
+};
+use pezsp_rpc::number::NumberOrHex;
+use pezsp_version::RuntimeVersion;
+
+/// RPC methods of Bizinikiwi `system` namespace, that we are using.
+#[rpc(client, client_bounds(C: Chain), namespace = "system")]
+pub(crate) trait BizinikiwiSystem<C> {
+	/// Return node health.
+	#[method(name = "health")]
+	async fn health(&self) -> RpcResult<Health>;
+	/// Return system properties.
+	#[method(name = "properties")]
+	async fn properties(&self) -> RpcResult<pezsc_chain_spec::Properties>;
+}
+
+/// RPC methods of Bizinikiwi `chain` namespace, that we are using.
+#[rpc(client, client_bounds(C: Chain), namespace = "chain")]
+pub(crate) trait BizinikiwiChain<C> {
+	/// Get block hash by its number.
+	#[method(name = "getBlockHash")]
+	async fn block_hash(&self, block_number: Option<C::BlockNumber>) -> RpcResult<C::Hash>;
+	/// Return block header by its hash.
+	#[method(name = "getHeader")]
+	async fn header(&self, block_hash: Option<C::Hash>) -> RpcResult<C::Header>;
+	/// Return best finalized block hash.
+	#[method(name = "getFinalizedHead")]
+	async fn finalized_head(&self) -> RpcResult<C::Hash>;
+	/// Return signed block (with justifications) by its hash.
+	#[method(name = "getBlock")]
+	async fn block(&self, block_hash: Option<C::Hash>) -> RpcResult<C::SignedBlock>;
+	/// Subscribe to best headers.
+	#[subscription(
+		name = "subscribeNewHeads" => "newHead",
+		unsubscribe = "unsubscribeNewHeads",
+		item = C::Header
+	)]
+	async fn subscribe_new_heads(&self);
+	/// Subscribe to finalized headers.
+	#[subscription(
+		name = "subscribeFinalizedHeads" => "finalizedHead",
+		unsubscribe = "unsubscribeFinalizedHeads",
+		item = C::Header
+	)]
+	async fn subscribe_finalized_heads(&self);
+}
+
+/// RPC methods of Bizinikiwi `author` namespace, that we are using.
+#[rpc(client, client_bounds(C: Chain), namespace = "author")]
+pub(crate) trait BizinikiwiAuthor<C> {
+	/// Submit extrinsic to the transaction pool.
+	#[method(name = "submitExtrinsic")]
+	async fn submit_extrinsic(&self, extrinsic: Bytes) -> RpcResult<C::Hash>;
+	/// Return vector of pending extrinsics from the transaction pool.
+	#[method(name = "pendingExtrinsics")]
+	async fn pending_extrinsics(&self) -> RpcResult<Vec<Bytes>>;
+	/// Submit and watch for extrinsic state.
+	#[subscription(name = "submitAndWatchExtrinsic", unsubscribe = "unwatchExtrinsic", item = TransactionStatusOf<C>)]
+	async fn submit_and_watch_extrinsic(&self, extrinsic: Bytes);
+}
+
+/// RPC methods of Bizinikiwi `state` namespace, that we are using.
+#[rpc(client, client_bounds(C: Chain), namespace = "state")]
+pub(crate) trait BizinikiwiState<C> {
+	/// Get current runtime version.
+	#[method(name = "getRuntimeVersion")]
+	async fn runtime_version(&self) -> RpcResult<RuntimeVersion>;
+	/// Call given runtime method.
+	#[method(name = "call")]
+	async fn call(
+		&self,
+		method: String,
+		data: Bytes,
+		at_block: Option<C::Hash>,
+	) -> RpcResult<Bytes>;
+	/// Get value of the runtime storage.
+	#[method(name = "getStorage")]
+	async fn storage(
+		&self,
+		key: StorageKey,
+		at_block: Option<C::Hash>,
+	) -> RpcResult<Option<StorageData>>;
+	/// Get proof of the runtime storage value.
+	#[method(name = "getReadProof")]
+	async fn prove_storage(
+		&self,
+		keys: Vec<StorageKey>,
+		hash: Option<C::Hash>,
+	) -> RpcResult<ReadProof<C::Hash>>;
+}
+
+/// RPC methods of Bizinikiwi `grandpa` namespace, that we are using.
+#[rpc(client, client_bounds(C: ChainWithGrandpa), namespace = "grandpa")]
+pub(crate) trait BizinikiwiGrandpa<C> {
+	/// Subscribe to GRANDPA justifications.
+	#[subscription(name = "subscribeJustifications", unsubscribe = "unsubscribeJustifications", item = Bytes)]
+	async fn subscribe_justifications(&self);
+}
+
+// TODO: Use `ChainWithBeefy` instead of `Chain` after #1606 is merged
+/// RPC methods of Bizinikiwi `beefy` namespace, that we are using.
+#[rpc(client, client_bounds(C: Chain), namespace = "beefy")]
+pub(crate) trait BizinikiwiBeefy<C> {
+	/// Subscribe to BEEFY justifications.
+	#[subscription(name = "subscribeJustifications", unsubscribe = "unsubscribeJustifications", item = Bytes)]
+	async fn subscribe_justifications(&self);
+}
+
+/// RPC methods of Bizinikiwi `system` frame pezpallet, that we are using.
+#[rpc(client, client_bounds(C: Chain), namespace = "system")]
+pub(crate) trait BizinikiwiFrameSystem<C> {
+	/// Return index of next account transaction.
+	#[method(name = "accountNextIndex")]
+	async fn account_next_index(&self, account_id: C::AccountId) -> RpcResult<C::Nonce>;
+}
+
+/// RPC methods of Bizinikiwi `pezpallet_transaction_payment` frame pezpallet, that we are using.
+#[rpc(client, client_bounds(C: Chain), namespace = "payment")]
+pub(crate) trait BizinikiwiTransactionPayment<C> {
+	/// Query transaction fee details.
+	#[method(name = "queryFeeDetails")]
+	async fn fee_details(
+		&self,
+		extrinsic: Bytes,
+		at_block: Option<C::Hash>,
+	) -> RpcResult<FeeDetails<NumberOrHex>>;
+}

pezbridges/relays/client-bizinikiwi/src/client/subscription.rs

@@ -0,0 +1,238 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::error::Result as ClientResult;
+
+use async_std::{
+	channel::{bounded, Receiver, Sender},
+	stream::StreamExt,
+};
+use futures::{FutureExt, Stream};
+use pezsp_runtime::DeserializeOwned;
+use std::{
+	fmt::Debug,
+	pin::Pin,
+	result::Result as StdResult,
+	task::{Context, Poll},
+};
+
+/// Once channel reaches this capacity, the subscription breaks.
+const CHANNEL_CAPACITY: usize = 128;
+
+/// Structure describing a stream.
+#[derive(Clone)]
+pub struct StreamDescription {
+	stream_name: String,
+	chain_name: String,
+}
+
+impl StreamDescription {
+	/// Create a new instance of `StreamDescription`.
+	pub fn new(stream_name: String, chain_name: String) -> Self {
+		Self { stream_name, chain_name }
+	}
+
+	/// Get a stream description.
+	fn get(&self) -> String {
+		format!("{} stream of {}", self.stream_name, self.chain_name)
+	}
+}
+
+/// Chainable stream that transforms items of type `Result<T, E>` to items of type `T`.
+///
+/// If it encounters an item of type `Err`, it returns `Poll::Ready(None)`
+/// and terminates the underlying stream.
+struct Unwrap<S: Stream<Item = StdResult<T, E>>, T, E> {
+	desc: StreamDescription,
+	stream: Option<S>,
+}
+
+impl<S: Stream<Item = StdResult<T, E>>, T, E> Unwrap<S, T, E> {
+	/// Create a new instance of `Unwrap`.
+	pub fn new(desc: StreamDescription, stream: S) -> Self {
+		Self { desc, stream: Some(stream) }
+	}
+}
+
+impl<S: Stream<Item = StdResult<T, E>> + Unpin, T: DeserializeOwned, E: Debug> Stream
+	for Unwrap<S, T, E>
+{
+	type Item = T;
+
+	fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+		Poll::Ready(match self.stream.as_mut() {
+			Some(subscription) => match futures::ready!(Pin::new(subscription).poll_next(cx)) {
+				Some(Ok(item)) => Some(item),
+				Some(Err(e)) => {
+					self.stream.take();
+					tracing::debug!(
+						target: "bridge",
+						error=?e,
+						desc=%self.desc.get(),
+						"Returned with error. It may need to be restarted"
+					);
+					None
+				},
+				None => {
+					self.stream.take();
+					tracing::debug!(
+						target: "bridge",
+						desc=%self.desc.get(),
+						"Returned `None`. It may need to be restarted"
+					);
+					None
+				},
+			},
+			None => None,
+		})
+	}
+}
+
+/// Subscription factory that produces subscriptions, sharing the same background thread.
+#[derive(Clone)]
+pub struct SubscriptionBroadcaster<T> {
+	desc: StreamDescription,
+	subscribers_sender: Sender<Sender<T>>,
+}
+
+impl<T: 'static + Clone + DeserializeOwned + Send> SubscriptionBroadcaster<T> {
+	/// Create new subscription factory.
+	pub fn new(subscription: Subscription<T>) -> StdResult<Self, Subscription<T>> {
+		// It doesn't make sense to further broadcast a broadcasted subscription.
+		if subscription.is_broadcasted {
+			return Err(subscription);
+		}
+
+		let desc = subscription.desc().clone();
+		let (subscribers_sender, subscribers_receiver) = bounded(CHANNEL_CAPACITY);
+		async_std::task::spawn(background_worker(subscription, subscribers_receiver));
+		Ok(Self { desc, subscribers_sender })
+	}
+
+	/// Produce new subscription.
+	pub async fn subscribe(&self) -> ClientResult<Subscription<T>> {
+		let (items_sender, items_receiver) = bounded(CHANNEL_CAPACITY);
+		self.subscribers_sender.try_send(items_sender)?;
+
+		Ok(Subscription::new_broadcasted(self.desc.clone(), items_receiver))
+	}
+}
+
+/// Subscription to some chain events.
+pub struct Subscription<T> {
+	desc: StreamDescription,
+	subscription: Box<dyn Stream<Item = T> + Unpin + Send>,
+	is_broadcasted: bool,
+}
+
+impl<T: 'static + Clone + DeserializeOwned + Send> Subscription<T> {
+	/// Create new forwarded subscription.
+	pub fn new_forwarded(
+		desc: StreamDescription,
+		subscription: impl Stream<Item = StdResult<T, serde_json::Error>> + Unpin + Send + 'static,
+	) -> Self {
+		Self {
+			desc: desc.clone(),
+			subscription: Box::new(Unwrap::new(desc, subscription)),
+			is_broadcasted: false,
+		}
+	}
+
+	/// Create new broadcasted subscription.
+	pub fn new_broadcasted(
+		desc: StreamDescription,
+		subscription: impl Stream<Item = T> + Unpin + Send + 'static,
+	) -> Self {
+		Self { desc, subscription: Box::new(subscription), is_broadcasted: true }
+	}
+
+	/// Get the description of the underlying stream
+	pub fn desc(&self) -> &StreamDescription {
+		&self.desc
+	}
+}
+
+impl<T> Stream for Subscription<T> {
+	type Item = T;
+
+	fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+		Poll::Ready(futures::ready!(Pin::new(&mut self.subscription).poll_next(cx)))
+	}
+}
+
+/// Background worker that is executed in tokio context as `jsonrpsee` requires.
+///
+/// This task may exit under some circumstances. It'll send the correspondent
+/// message (`Err` or `None`) to all known listeners. Also, when it stops, all
+/// subsequent reads and new subscribers will get the connection error (`ChannelError`).
+async fn background_worker<T: 'static + Clone + DeserializeOwned + Send>(
+	mut subscription: Subscription<T>,
+	mut subscribers_receiver: Receiver<Sender<T>>,
+) {
+	fn log_task_exit(desc: &StreamDescription, reason: &str) {
+		tracing::debug!(
+			target: "bridge",
+			desc=%desc.get(),
+			%reason,
+			"Background task of subscription broadcaster has stopped"
+		);
+	}
+
+	// wait for first subscriber until actually starting subscription
+	let subscriber = match subscribers_receiver.next().await {
+		Some(subscriber) => subscriber,
+		None => {
+			// it means that the last subscriber/factory has been dropped, so we need to
+			// exit too
+			return log_task_exit(subscription.desc(), "client has stopped");
+		},
+	};
+
+	// actually subscribe
+	let mut subscribers = vec![subscriber];
+
+	// start listening for new items and receivers
+	loop {
+		futures::select! {
+			subscriber = subscribers_receiver.next().fuse() => {
+				match subscriber {
+					Some(subscriber) => subscribers.push(subscriber),
+					None => {
+						// it means that the last subscriber/factory has been dropped, so we need to
+						// exit too
+						return log_task_exit(subscription.desc(), "client has stopped")
+					},
+				}
+			},
+			maybe_item = subscription.subscription.next().fuse() => {
+				match maybe_item {
+					Some(item) => {
+						// notify subscribers
+						subscribers.retain(|subscriber| {
+							let send_result = subscriber.try_send(item.clone());
+							send_result.is_ok()
+						});
+					}
+					None => {
+						// The underlying client has dropped, so we can't do anything here
+						// and need to stop the task.
+						return log_task_exit(subscription.desc(), "stream has finished");
+					}
+				}
+			},
+		}
+	}
+}

pezbridges/relays/client-bizinikiwi/src/client/traits.rs

@@ -0,0 +1,234 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{
+	error::{Error, Result},
+	AccountIdOf, AccountKeyPairOf, BlockNumberOf, Chain, ChainWithGrandpa, ChainWithTransactions,
+	HashOf, HeaderIdOf, HeaderOf, NonceOf, SignedBlockOf, SimpleRuntimeVersion, Subscription,
+	TransactionTracker, UnsignedTransaction,
+};
+
+use async_trait::async_trait;
+use pezbp_runtime::{StorageDoubleMapKeyProvider, StorageMapKeyProvider};
+use codec::{Decode, Encode};
+use pezframe_support::weights::Weight;
+use pezsp_core::{
+	storage::{StorageData, StorageKey},
+	Bytes, Pair,
+};
+use pezsp_runtime::{traits::Header as _, transaction_validity::TransactionValidity};
+use pezsp_trie::StorageProof;
+use pezsp_version::RuntimeVersion;
+use std::fmt::Debug;
+
+/// Relay uses the `Client` to communicate with the node, connected to Bizinikiwi
+/// chain `C`.
+#[async_trait]
+pub trait Client<C: Chain>: 'static + Send + Sync + Clone + Debug {
+	/// Returns error if client has no connected peers or it believes it is far
+	/// behind the chain tip.
+	async fn ensure_synced(&self) -> Result<()>;
+	/// Reconnects the client.
+	async fn reconnect(&self) -> Result<()>;
+
+	/// Return hash of the genesis block.
+	fn genesis_hash(&self) -> HashOf<C>;
+	/// Get header hash by number.
+	async fn header_hash_by_number(&self, number: BlockNumberOf<C>) -> Result<HashOf<C>>;
+	/// Get header by hash.
+	async fn header_by_hash(&self, hash: HashOf<C>) -> Result<HeaderOf<C>>;
+	/// Get header by number.
+	async fn header_by_number(&self, number: BlockNumberOf<C>) -> Result<HeaderOf<C>> {
+		self.header_by_hash(self.header_hash_by_number(number).await?).await
+	}
+	/// Get block by hash.
+	async fn block_by_hash(&self, hash: HashOf<C>) -> Result<SignedBlockOf<C>>;
+
+	/// Get best finalized header hash.
+	async fn best_finalized_header_hash(&self) -> Result<HashOf<C>>;
+	/// Get best finalized header number.
+	async fn best_finalized_header_number(&self) -> Result<BlockNumberOf<C>> {
+		Ok(*self.best_finalized_header().await?.number())
+	}
+	/// Get best finalized header.
+	async fn best_finalized_header(&self) -> Result<HeaderOf<C>> {
+		self.header_by_hash(self.best_finalized_header_hash().await?).await
+	}
+
+	/// Get best header.
+	async fn best_header(&self) -> Result<HeaderOf<C>>;
+	/// Get best header hash.
+	async fn best_header_hash(&self) -> Result<HashOf<C>> {
+		Ok(self.best_header().await?.hash())
+	}
+
+	/// Subscribe to new best headers.
+	async fn subscribe_best_headers(&self) -> Result<Subscription<HeaderOf<C>>>;
+	/// Subscribe to new finalized headers.
+	async fn subscribe_finalized_headers(&self) -> Result<Subscription<HeaderOf<C>>>;
+
+	/// Subscribe to GRANDPA finality justifications.
+	async fn subscribe_grandpa_finality_justifications(&self) -> Result<Subscription<Bytes>>
+	where
+		C: ChainWithGrandpa;
+	/// Generates a proof of key ownership for the given authority in the given set.
+	async fn generate_grandpa_key_ownership_proof(
+		&self,
+		at: HashOf<C>,
+		set_id: pezsp_consensus_grandpa::SetId,
+		authority_id: pezsp_consensus_grandpa::AuthorityId,
+	) -> Result<Option<pezsp_consensus_grandpa::OpaqueKeyOwnershipProof>>;
+
+	/// Subscribe to BEEFY finality justifications.
+	async fn subscribe_beefy_finality_justifications(&self) -> Result<Subscription<Bytes>>;
+
+	/// Return `tokenDecimals` property from the set of chain properties.
+	async fn token_decimals(&self) -> Result<Option<u64>>;
+	/// Get runtime version of the connected chain.
+	async fn runtime_version(&self) -> Result<RuntimeVersion>;
+	/// Get partial runtime version, to use when signing transactions.
+	async fn simple_runtime_version(&self) -> Result<SimpleRuntimeVersion>;
+	/// Returns `true` if version guard can be started.
+	///
+	/// There's no reason to run version guard when version mode is set to `Auto`. It can
+	/// lead to relay shutdown when chain is upgraded, even though we have explicitly
+	/// said that we don't want to shutdown.
+	fn can_start_version_guard(&self) -> bool;
+
+	/// Read raw value from runtime storage.
+	async fn raw_storage_value(
+		&self,
+		at: HashOf<C>,
+		storage_key: StorageKey,
+	) -> Result<Option<StorageData>>;
+	/// Read and decode value from runtime storage.
+	async fn storage_value<T: Decode + 'static>(
+		&self,
+		at: HashOf<C>,
+		storage_key: StorageKey,
+	) -> Result<Option<T>> {
+		self.raw_storage_value(at, storage_key.clone())
+			.await?
+			.map(|encoded_value| {
+				T::decode(&mut &encoded_value.0[..]).map_err(|e| {
+					Error::failed_to_read_storage_value::<C>(at, storage_key, e.into())
+				})
+			})
+			.transpose()
+	}
+	/// Read and decode value from runtime storage map.
+	///
+	/// `pezpallet_prefix` is the name of the pezpallet (used in `construct_runtime`), which
+	/// "contains" the storage map.
+	async fn storage_map_value<T: StorageMapKeyProvider>(
+		&self,
+		at: HashOf<C>,
+		pezpallet_prefix: &str,
+		storage_key: &T::Key,
+	) -> Result<Option<T::Value>> {
+		self.storage_value(at, T::final_key(pezpallet_prefix, storage_key)).await
+	}
+	/// Read and decode value from runtime storage double map.
+	///
+	/// `pezpallet_prefix` is the name of the pezpallet (used in `construct_runtime`), which
+	/// "contains" the storage double map.
+	async fn storage_double_map_value<T: StorageDoubleMapKeyProvider>(
+		&self,
+		at: HashOf<C>,
+		pezpallet_prefix: &str,
+		key1: &T::Key1,
+		key2: &T::Key2,
+	) -> Result<Option<T::Value>> {
+		self.storage_value(at, T::final_key(pezpallet_prefix, key1, key2)).await
+	}
+
+	/// Returns pending extrinsics from transaction pool.
+	async fn pending_extrinsics(&self) -> Result<Vec<Bytes>>;
+	/// Submit unsigned extrinsic for inclusion in a block.
+	///
+	/// Note: The given transaction needs to be SCALE encoded beforehand.
+	async fn submit_unsigned_extrinsic(&self, transaction: Bytes) -> Result<HashOf<C>>;
+	/// Submit an extrinsic signed by given account.
+	///
+	/// All calls of this method are synchronized, so there can't be more than one active
+	/// `submit_signed_extrinsic()` call. This guarantees that no nonces collision may happen
+	/// if all client instances are clones of the same initial `Client`.
+	///
+	/// Note: The given transaction needs to be SCALE encoded beforehand.
+	async fn submit_signed_extrinsic(
+		&self,
+		signer: &AccountKeyPairOf<C>,
+		prepare_extrinsic: impl FnOnce(HeaderIdOf<C>, NonceOf<C>) -> Result<UnsignedTransaction<C>>
+			+ Send
+			+ 'static,
+	) -> Result<HashOf<C>>
+	where
+		C: ChainWithTransactions,
+		AccountIdOf<C>: From<<AccountKeyPairOf<C> as Pair>::Public>;
+	/// Does exactly the same as `submit_signed_extrinsic`, but keeps watching for extrinsic status
+	/// after submission.
+	async fn submit_and_watch_signed_extrinsic(
+		&self,
+		signer: &AccountKeyPairOf<C>,
+		prepare_extrinsic: impl FnOnce(HeaderIdOf<C>, NonceOf<C>) -> Result<UnsignedTransaction<C>>
+			+ Send
+			+ 'static,
+	) -> Result<TransactionTracker<C, Self>>
+	where
+		C: ChainWithTransactions,
+		AccountIdOf<C>: From<<AccountKeyPairOf<C> as Pair>::Public>;
+	/// Validate transaction at given block.
+	async fn validate_transaction<SignedTransaction: Encode + Send + 'static>(
+		&self,
+		at: HashOf<C>,
+		transaction: SignedTransaction,
+	) -> Result<TransactionValidity>;
+	/// Returns weight of the given transaction.
+	async fn estimate_extrinsic_weight<SignedTransaction: Encode + Send + 'static>(
+		&self,
+		at: HashOf<C>,
+		transaction: SignedTransaction,
+	) -> Result<Weight>;
+
+	/// Execute runtime call at given block.
+	async fn raw_state_call<Args: Encode + Send>(
+		&self,
+		at: HashOf<C>,
+		method: String,
+		arguments: Args,
+	) -> Result<Bytes>;
+	/// Execute runtime call at given block, provided the input and output types.
+	/// It also performs the input encode and output decode.
+	async fn state_call<Args: Encode + Send, Ret: Decode>(
+		&self,
+		at: HashOf<C>,
+		method: String,
+		arguments: Args,
+	) -> Result<Ret> {
+		let encoded_arguments = arguments.encode();
+		let encoded_output = self.raw_state_call(at, method.clone(), arguments).await?;
+		Ret::decode(&mut &encoded_output.0[..]).map_err(|e| {
+			Error::failed_state_call::<C>(at, method, Bytes(encoded_arguments), e.into())
+		})
+	}
+
+	/// Returns storage proof of given storage keys and state root.
+	async fn prove_storage(
+		&self,
+		at: HashOf<C>,
+		keys: Vec<StorageKey>,
+	) -> Result<(StorageProof, HashOf<C>)>;
+}

pezbridges/relays/client-bizinikiwi/src/error.rs

@@ -0,0 +1,431 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Bizinikiwi node RPC errors.
+
+use crate::{BlockNumberOf, Chain, HashOf, SimpleRuntimeVersion};
+use bp_header_pez_chain::SubmitFinalityProofCallExtras;
+use bp_pezkuwi_core::teyrchains::ParaId;
+use jsonrpsee::core::ClientError as RpcError;
+use relay_utils::MaybeConnectionError;
+use pezsc_rpc_api::system::Health;
+use pezsp_core::{storage::StorageKey, Bytes};
+use pezsp_runtime::transaction_validity::TransactionValidityError;
+use thiserror::Error;
+
+/// Result type used by Bizinikiwi client.
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Errors that can occur only when interacting with
+/// a Bizinikiwi node through RPC.
+#[derive(Error, Debug)]
+pub enum Error {
+	/// IO error.
+	#[error("IO error: {0}")]
+	Io(#[from] std::io::Error),
+	/// An error that can occur when making a request to
+	/// an JSON-RPC server.
+	#[error("RPC error: {0}")]
+	RpcError(#[from] RpcError),
+	/// The response from the server could not be SCALE decoded.
+	#[error("Response parse failed: {0}")]
+	ResponseParseFailed(#[from] codec::Error),
+	/// Internal channel error - communication channel is either closed, or full.
+	/// It can be solved with reconnect.
+	#[error("Internal communication channel error: {0:?}.")]
+	ChannelError(String),
+	/// Required teyrchain head is not present at the relay chain.
+	#[error("Teyrchain {0:?} head {1} is missing from the relay chain storage.")]
+	MissingRequiredTeyrchainHead(ParaId, u64),
+	/// Failed to find finality proof for the given header.
+	#[error("Failed to find finality proof for header {0}.")]
+	FinalityProofNotFound(u64),
+	/// The client we're connected to is not synced, so we can't rely on its state.
+	#[error("Bizinikiwi client is not synced {0}.")]
+	ClientNotSynced(Health),
+	/// Failed to get system health.
+	#[error("Failed to get system health of {chain} node: {error:?}.")]
+	FailedToGetSystemHealth {
+		/// Name of the chain where the error has happened.
+		chain: String,
+		/// Underlying error.
+		error: Box<Error>,
+	},
+	/// Failed to read best finalized header hash from given chain.
+	#[error("Failed to read best finalized header hash of {chain}: {error:?}.")]
+	FailedToReadBestFinalizedHeaderHash {
+		/// Name of the chain where the error has happened.
+		chain: String,
+		/// Underlying error.
+		error: Box<Error>,
+	},
+	/// Failed to read best finalized header from given chain.
+	#[error("Failed to read best header of {chain}: {error:?}.")]
+	FailedToReadBestHeader {
+		/// Name of the chain where the error has happened.
+		chain: String,
+		/// Underlying error.
+		error: Box<Error>,
+	},
+	/// Failed to read header hash by number from given chain.
+	#[error("Failed to read header hash by number {number} of {chain}: {error:?}.")]
+	FailedToReadHeaderHashByNumber {
+		/// Name of the chain where the error has happened.
+		chain: String,
+		/// Number of the header we've tried to read.
+		number: String,
+		/// Underlying error.
+		error: Box<Error>,
+	},
+	/// Failed to read header by hash from given chain.
+	#[error("Failed to read header {hash} of {chain}: {error:?}.")]
+	FailedToReadHeaderByHash {
+		/// Name of the chain where the error has happened.
+		chain: String,
+		/// Hash of the header we've tried to read.
+		hash: String,
+		/// Underlying error.
+		error: Box<Error>,
+	},
+	/// Failed to read block by hash from given chain.
+	#[error("Failed to read block {hash} of {chain}: {error:?}.")]
+	FailedToReadBlockByHash {
+		/// Name of the chain where the error has happened.
+		chain: String,
+		/// Hash of the header we've tried to read.
+		hash: String,
+		/// Underlying error.
+		error: Box<Error>,
+	},
+	/// Failed to read sotrage value at given chain.
+	#[error("Failed to read storage value {key:?} at {chain}: {error:?}.")]
+	FailedToReadStorageValue {
+		/// Name of the chain where the error has happened.
+		chain: String,
+		/// Hash of the block we've tried to read value from.
+		hash: String,
+		/// Runtime storage key
+		key: StorageKey,
+		/// Underlying error.
+		error: Box<Error>,
+	},
+	/// Failed to read runtime version of given chain.
+	#[error("Failed to read runtime version of {chain}: {error:?}.")]
+	FailedToReadRuntimeVersion {
+		/// Name of the chain where the error has happened.
+		chain: String,
+		/// Underlying error.
+		error: Box<Error>,
+	},
+	/// Failed to get pending extrinsics.
+	#[error("Failed to get pending extrinsics of {chain}: {error:?}.")]
+	FailedToGetPendingExtrinsics {
+		/// Name of the chain where the error has happened.
+		chain: String,
+		/// Underlying error.
+		error: Box<Error>,
+	},
+	/// Failed to submit transaction.
+	#[error("Failed to submit {chain} transaction: {error:?}.")]
+	FailedToSubmitTransaction {
+		/// Name of the chain where the error has happened.
+		chain: String,
+		/// Underlying error.
+		error: Box<Error>,
+	},
+	/// Runtime call has failed.
+	#[error("Runtime call {method} with arguments {arguments:?} of chain {chain} at {hash} has failed: {error:?}.")]
+	FailedStateCall {
+		/// Name of the chain where the error has happened.
+		chain: String,
+		/// Hash of the block we've tried to call at.
+		hash: String,
+		/// Runtime API method.
+		method: String,
+		/// Encoded method arguments.
+		arguments: Bytes,
+		/// Underlying error.
+		error: Box<Error>,
+	},
+	/// Failed to prove storage keys.
+	#[error("Failed to prove storage keys {storage_keys:?} of {chain} at {hash}: {error:?}.")]
+	FailedToProveStorage {
+		/// Name of the chain where the error has happened.
+		chain: String,
+		/// Hash of the block we've tried to prove keys at.
+		hash: String,
+		/// Storage keys we have tried to prove.
+		storage_keys: Vec<StorageKey>,
+		/// Underlying error.
+		error: Box<Error>,
+	},
+	/// Failed to subscribe to GRANDPA justifications stream.
+	#[error("Failed to subscribe to {chain} best headers: {error:?}.")]
+	FailedToSubscribeBestHeaders {
+		/// Name of the chain where the error has happened.
+		chain: String,
+		/// Underlying error.
+		error: Box<Error>,
+	},
+	/// Failed to subscribe to GRANDPA justifications stream.
+	#[error("Failed to subscribe to {chain} finalized headers: {error:?}.")]
+	FailedToSubscribeFinalizedHeaders {
+		/// Name of the chain where the error has happened.
+		chain: String,
+		/// Underlying error.
+		error: Box<Error>,
+	},
+	/// Failed to subscribe to GRANDPA justifications stream.
+	#[error("Failed to subscribe to {chain} justifications: {error:?}.")]
+	FailedToSubscribeJustifications {
+		/// Name of the chain where the error has happened.
+		chain: String,
+		/// Underlying error.
+		error: Box<Error>,
+	},
+	/// Headers of the chain are finalized out of order. Maybe chain has been
+	/// restarted?
+	#[error("Finalized headers of {chain} are unordered: previously finalized {prev_number} vs new {next_number}")]
+	UnorderedFinalizedHeaders {
+		/// Name of the chain where the error has happened.
+		chain: String,
+		/// Previously finalized header number.
+		prev_number: String,
+		/// New finalized header number.
+		next_number: String,
+	},
+	/// The bridge pezpallet is halted and all transactions will be rejected.
+	#[error("Bridge pezpallet is halted.")]
+	BridgePalletIsHalted,
+	/// The bridge pezpallet is not yet initialized and all transactions will be rejected.
+	#[error("Bridge pezpallet is not initialized.")]
+	BridgePalletIsNotInitialized,
+	/// The Bizinikiwi transaction is invalid.
+	#[error("Bizinikiwi transaction is invalid: {0:?}")]
+	TransactionInvalid(#[from] TransactionValidityError),
+	/// The client is configured to use newer runtime version than the connected chain uses.
+	/// The client will keep waiting until chain is upgraded to given version.
+	#[error("Waiting for {chain} runtime upgrade: expected {expected:?} actual {actual:?}")]
+	WaitingForRuntimeUpgrade {
+		/// Name of the chain where the error has happened.
+		chain: String,
+		/// Expected runtime version.
+		expected: SimpleRuntimeVersion,
+		/// Actual runtime version.
+		actual: SimpleRuntimeVersion,
+	},
+	/// Finality proof submission exceeds size and/or weight limits.
+	#[error("Finality proof submission exceeds limits: {extras:?}")]
+	FinalityProofWeightLimitExceeded {
+		/// Finality proof submission extras.
+		extras: SubmitFinalityProofCallExtras,
+	},
+	/// Custom logic error.
+	#[error("{0}")]
+	Custom(String),
+}
+
+impl From<tokio::task::JoinError> for Error {
+	fn from(error: tokio::task::JoinError) -> Self {
+		Error::ChannelError(format!("failed to wait tokio task: {error}"))
+	}
+}
+
+impl<T> From<async_std::channel::TrySendError<T>> for Error {
+	fn from(error: async_std::channel::TrySendError<T>) -> Self {
+		Error::ChannelError(format!("`try_send` has failed: {error:?}"))
+	}
+}
+
+impl From<async_std::channel::RecvError> for Error {
+	fn from(error: async_std::channel::RecvError) -> Self {
+		Error::ChannelError(format!("`recv` has failed: {error:?}"))
+	}
+}
+
+impl Error {
+	/// Box the error.
+	pub fn boxed(self) -> Box<Self> {
+		Box::new(self)
+	}
+
+	/// Returns nested error reference.
+	pub fn nested(&self) -> Option<&Self> {
+		match *self {
+			Self::FailedToReadBestFinalizedHeaderHash { ref error, .. } => Some(&**error),
+			Self::FailedToReadBestHeader { ref error, .. } => Some(&**error),
+			Self::FailedToReadHeaderHashByNumber { ref error, .. } => Some(&**error),
+			Self::FailedToReadHeaderByHash { ref error, .. } => Some(&**error),
+			Self::FailedToReadBlockByHash { ref error, .. } => Some(&**error),
+			Self::FailedToReadStorageValue { ref error, .. } => Some(&**error),
+			Self::FailedToReadRuntimeVersion { ref error, .. } => Some(&**error),
+			Self::FailedToGetPendingExtrinsics { ref error, .. } => Some(&**error),
+			Self::FailedToSubmitTransaction { ref error, .. } => Some(&**error),
+			Self::FailedStateCall { ref error, .. } => Some(&**error),
+			Self::FailedToProveStorage { ref error, .. } => Some(&**error),
+			Self::FailedToGetSystemHealth { ref error, .. } => Some(&**error),
+			Self::FailedToSubscribeBestHeaders { ref error, .. } => Some(&**error),
+			Self::FailedToSubscribeFinalizedHeaders { ref error, .. } => Some(&**error),
+			Self::FailedToSubscribeJustifications { ref error, .. } => Some(&**error),
+			_ => None,
+		}
+	}
+
+	/// Constructs `FailedToReadHeaderHashByNumber` variant.
+	pub fn failed_to_read_header_hash_by_number<C: Chain>(
+		number: BlockNumberOf<C>,
+		e: Error,
+	) -> Self {
+		Error::FailedToReadHeaderHashByNumber {
+			chain: C::NAME.into(),
+			number: format!("{number}"),
+			error: e.boxed(),
+		}
+	}
+
+	/// Constructs `FailedToReadHeaderByHash` variant.
+	pub fn failed_to_read_header_by_hash<C: Chain>(hash: HashOf<C>, e: Error) -> Self {
+		Error::FailedToReadHeaderByHash {
+			chain: C::NAME.into(),
+			hash: format!("{hash}"),
+			error: e.boxed(),
+		}
+	}
+
+	/// Constructs `FailedToReadBlockByHash` variant.
+	pub fn failed_to_read_block_by_hash<C: Chain>(hash: HashOf<C>, e: Error) -> Self {
+		Error::FailedToReadHeaderByHash {
+			chain: C::NAME.into(),
+			hash: format!("{hash}"),
+			error: e.boxed(),
+		}
+	}
+
+	/// Constructs `FailedToReadBestFinalizedHeaderHash` variant.
+	pub fn failed_to_read_best_finalized_header_hash<C: Chain>(e: Error) -> Self {
+		Error::FailedToReadBestFinalizedHeaderHash { chain: C::NAME.into(), error: e.boxed() }
+	}
+
+	/// Constructs `FailedToReadBestHeader` variant.
+	pub fn failed_to_read_best_header<C: Chain>(e: Error) -> Self {
+		Error::FailedToReadBestHeader { chain: C::NAME.into(), error: e.boxed() }
+	}
+
+	/// Constructs `FailedToReadRuntimeVersion` variant.
+	pub fn failed_to_read_runtime_version<C: Chain>(e: Error) -> Self {
+		Error::FailedToReadRuntimeVersion { chain: C::NAME.into(), error: e.boxed() }
+	}
+
+	/// Constructs `FailedToReadStorageValue` variant.
+	pub fn failed_to_read_storage_value<C: Chain>(
+		at: HashOf<C>,
+		key: StorageKey,
+		e: Error,
+	) -> Self {
+		Error::FailedToReadStorageValue {
+			chain: C::NAME.into(),
+			hash: format!("{at}"),
+			key,
+			error: e.boxed(),
+		}
+	}
+
+	/// Constructs `FailedToGetPendingExtrinsics` variant.
+	pub fn failed_to_get_pending_extrinsics<C: Chain>(e: Error) -> Self {
+		Error::FailedToGetPendingExtrinsics { chain: C::NAME.into(), error: e.boxed() }
+	}
+
+	/// Constructs `FailedToSubmitTransaction` variant.
+	pub fn failed_to_submit_transaction<C: Chain>(e: Error) -> Self {
+		Error::FailedToSubmitTransaction { chain: C::NAME.into(), error: e.boxed() }
+	}
+
+	/// Constructs `FailedStateCall` variant.
+	pub fn failed_state_call<C: Chain>(
+		at: HashOf<C>,
+		method: String,
+		arguments: Bytes,
+		e: Error,
+	) -> Self {
+		Error::FailedStateCall {
+			chain: C::NAME.into(),
+			hash: format!("{at}"),
+			method,
+			arguments,
+			error: e.boxed(),
+		}
+	}
+
+	/// Constructs `FailedToProveStorage` variant.
+	pub fn failed_to_prove_storage<C: Chain>(
+		at: HashOf<C>,
+		storage_keys: Vec<StorageKey>,
+		e: Error,
+	) -> Self {
+		Error::FailedToProveStorage {
+			chain: C::NAME.into(),
+			hash: format!("{at}"),
+			storage_keys,
+			error: e.boxed(),
+		}
+	}
+
+	/// Constructs `FailedToGetSystemHealth` variant.
+	pub fn failed_to_get_system_health<C: Chain>(e: Error) -> Self {
+		Error::FailedToGetSystemHealth { chain: C::NAME.into(), error: e.boxed() }
+	}
+
+	/// Constructs `FailedToSubscribeBestHeaders` variant.
+	pub fn failed_to_subscribe_best_headers<C: Chain>(e: Error) -> Self {
+		Error::FailedToSubscribeBestHeaders { chain: C::NAME.into(), error: e.boxed() }
+	}
+
+	/// Constructs `FailedToSubscribeFinalizedHeaders` variant.
+	pub fn failed_to_subscribe_finalized_headers<C: Chain>(e: Error) -> Self {
+		Error::FailedToSubscribeFinalizedHeaders { chain: C::NAME.into(), error: e.boxed() }
+	}
+
+	/// Constructs `FailedToSubscribeJustifications` variant.
+	pub fn failed_to_subscribe_justification<C: Chain>(e: Error) -> Self {
+		Error::FailedToSubscribeJustifications { chain: C::NAME.into(), error: e.boxed() }
+	}
+
+	/// Constructs `Un`
+	pub fn unordered_finalized_headers<C: Chain>(
+		prev_number: BlockNumberOf<C>,
+		next_number: BlockNumberOf<C>,
+	) -> Self {
+		Error::UnorderedFinalizedHeaders {
+			chain: C::NAME.into(),
+			prev_number: format!("{}", prev_number),
+			next_number: format!("{}", next_number),
+		}
+	}
+}
+
+impl MaybeConnectionError for Error {
+	fn is_connection_error(&self) -> bool {
+		match *self {
+			Error::ChannelError(_) => true,
+			Error::RpcError(ref e) => {
+				matches!(*e, RpcError::Transport(_) | RpcError::RestartNeeded(_),)
+			},
+			Error::ClientNotSynced(_) => true,
+			Error::UnorderedFinalizedHeaders { .. } => true,
+			_ => self.nested().map(|e| e.is_connection_error()).unwrap_or(false),
+		}
+	}
+}

pezbridges/relays/client-bizinikiwi/src/guard.rs

@@ -0,0 +1,196 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Pezpallet provides a set of guard functions that are running in background threads
+//! and are aborting process if some condition fails.
+
+use crate::{error::Error, Chain, Client};
+
+use async_trait::async_trait;
+use pezsp_version::RuntimeVersion;
+use std::{
+	fmt::Display,
+	time::{Duration, Instant},
+};
+
+/// Guards environment.
+#[async_trait]
+pub trait Environment<C>: Send + Sync + 'static {
+	/// Error type.
+	type Error: Display + Send + Sync + 'static;
+
+	/// Return current runtime version.
+	async fn runtime_version(&mut self) -> Result<RuntimeVersion, Self::Error>;
+
+	/// Return current time.
+	fn now(&self) -> Instant {
+		Instant::now()
+	}
+
+	/// Sleep given amount of time.
+	async fn sleep(&mut self, duration: Duration) {
+		async_std::task::sleep(duration).await
+	}
+
+	/// Abort current process. Called when guard condition check fails.
+	async fn abort(&mut self) {
+		std::process::abort();
+	}
+}
+
+/// Abort when runtime spec version is different from specified.
+pub fn abort_on_spec_version_change<C: Chain>(
+	mut env: impl Environment<C>,
+	expected_spec_version: u32,
+) {
+	async_std::task::spawn(async move {
+		tracing::info!(
+			target: "bridge-guard",
+			node=%C::NAME,
+			%expected_spec_version,
+			"Starting spec_version guard."
+		);
+
+		loop {
+			let actual_spec_version = env.runtime_version().await;
+			match actual_spec_version {
+				Ok(version) if version.spec_version == expected_spec_version => (),
+				Ok(version) => {
+					tracing::error!(
+						target: "bridge-guard",
+						node=%C::NAME,
+						from=%expected_spec_version,
+						to=%version.spec_version,
+						"Runtime spec version has changed. Aborting relay"
+					);
+
+					env.abort().await;
+				},
+				Err(error) => tracing::warn!(
+					target: "bridge-guard",
+					%error,
+					node=%C::NAME,
+					"Failed to read runtime version. Relay may need to be stopped manually"
+				),
+			}
+
+			env.sleep(conditions_check_delay::<C>()).await;
+		}
+	});
+}
+
+/// Delay between conditions check.
+fn conditions_check_delay<C: Chain>() -> Duration {
+	C::AVERAGE_BLOCK_INTERVAL * (10 + rand::random::<u32>() % 10)
+}
+
+#[async_trait]
+impl<C: Chain, Clnt: Client<C>> Environment<C> for Clnt {
+	type Error = Error;
+
+	async fn runtime_version(&mut self) -> Result<RuntimeVersion, Self::Error> {
+		Client::<C>::runtime_version(self).await
+	}
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+	use super::*;
+	use crate::test_chain::TestChain;
+	use futures::{
+		channel::mpsc::{unbounded, UnboundedReceiver, UnboundedSender},
+		future::FutureExt,
+		stream::StreamExt,
+		SinkExt,
+	};
+
+	pub struct TestEnvironment {
+		pub runtime_version_rx: UnboundedReceiver<RuntimeVersion>,
+		pub slept_tx: UnboundedSender<()>,
+		pub aborted_tx: UnboundedSender<()>,
+	}
+
+	#[async_trait]
+	impl Environment<TestChain> for TestEnvironment {
+		type Error = Error;
+
+		async fn runtime_version(&mut self) -> Result<RuntimeVersion, Self::Error> {
+			Ok(self.runtime_version_rx.next().await.unwrap_or_default())
+		}
+
+		async fn sleep(&mut self, _duration: Duration) {
+			let _ = self.slept_tx.send(()).await;
+		}
+
+		async fn abort(&mut self) {
+			let _ = self.aborted_tx.send(()).await;
+			// simulate process abort :)
+			async_std::task::sleep(Duration::from_secs(60)).await;
+		}
+	}
+
+	#[test]
+	fn aborts_when_spec_version_is_changed() {
+		async_std::task::block_on(async {
+			let (
+				(mut runtime_version_tx, runtime_version_rx),
+				(slept_tx, mut slept_rx),
+				(aborted_tx, mut aborted_rx),
+			) = (unbounded(), unbounded(), unbounded());
+			abort_on_spec_version_change(
+				TestEnvironment { runtime_version_rx, slept_tx, aborted_tx },
+				0,
+			);
+
+			// client responds with wrong version
+			runtime_version_tx
+				.send(RuntimeVersion { spec_version: 42, ..Default::default() })
+				.await
+				.unwrap();
+
+			// then the `abort` function is called
+			aborted_rx.next().await;
+			// and we do not reach the `sleep` function call
+			assert!(slept_rx.next().now_or_never().is_none());
+		});
+	}
+
+	#[test]
+	fn does_not_aborts_when_spec_version_is_unchanged() {
+		async_std::task::block_on(async {
+			let (
+				(mut runtime_version_tx, runtime_version_rx),
+				(slept_tx, mut slept_rx),
+				(aborted_tx, mut aborted_rx),
+			) = (unbounded(), unbounded(), unbounded());
+			abort_on_spec_version_change(
+				TestEnvironment { runtime_version_rx, slept_tx, aborted_tx },
+				42,
+			);
+
+			// client responds with the same version
+			runtime_version_tx
+				.send(RuntimeVersion { spec_version: 42, ..Default::default() })
+				.await
+				.unwrap();
+
+			// then the `sleep` function is called
+			slept_rx.next().await;
+			// and the `abort` function is not called
+			assert!(aborted_rx.next().now_or_never().is_none());
+		});
+	}
+}

pezbridges/relays/client-bizinikiwi/src/lib.rs

@@ -0,0 +1,88 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Tools to interact with Bizinikiwi node using RPC methods.
+
+#![warn(missing_docs)]
+
+mod chain;
+mod client;
+mod error;
+mod sync_header;
+mod transaction_tracker;
+
+pub mod calls;
+pub mod guard;
+pub mod metrics;
+pub mod test_chain;
+
+use std::time::Duration;
+
+pub use crate::{
+	chain::{
+		AccountKeyPairOf, BlockWithJustification, CallOf, Chain, ChainWithBalances,
+		ChainWithGrandpa, ChainWithMessages, ChainWithRewards, ChainWithRuntimeVersion,
+		ChainWithTransactions, ChainWithUtilityPallet, FullRuntimeUtilityPallet,
+		MockedRuntimeUtilityPallet, RelayChain, SignParam, SignedBlockOf, Teyrchain,
+		TransactionStatusOf, UnsignedTransaction, UtilityPallet,
+	},
+	client::{
+		is_ancient_block, rpc_with_caching as new, ChainRuntimeVersion, Client,
+		OpaqueGrandpaAuthoritiesSet, RpcWithCachingClient, SimpleRuntimeVersion, StreamDescription,
+		Subscription, ANCIENT_BLOCK_THRESHOLD,
+	},
+	error::{Error, Result},
+	sync_header::SyncHeader,
+	transaction_tracker::TransactionTracker,
+};
+pub use pezbp_runtime::{
+	AccountIdOf, AccountPublicOf, BalanceOf, BlockNumberOf, Chain as ChainBase, HashOf, HeaderIdOf,
+	HeaderOf, NonceOf, SignatureOf, Teyrchain as TeyrchainBase, TransactionEra, TransactionEraOf,
+	UnderlyingChainProvider,
+};
+
+/// Bizinikiwi-over-websocket connection params.
+#[derive(Debug, Clone)]
+pub struct ConnectionParams {
+	/// Websocket endpoint URL.
+	pub uri: String,
+	/// Defined chain runtime version
+	pub chain_runtime_version: ChainRuntimeVersion,
+}
+
+impl Default for ConnectionParams {
+	fn default() -> Self {
+		ConnectionParams {
+			uri: "ws://localhost:9944".into(),
+			chain_runtime_version: ChainRuntimeVersion::Auto,
+		}
+	}
+}
+
+/// Returns stall timeout for relay loop.
+///
+/// Relay considers himself stalled if he has submitted transaction to the node, but it has not
+/// been mined for this period.
+pub fn transaction_stall_timeout(
+	mortality_period: Option<u32>,
+	average_block_interval: Duration,
+	default_stall_timeout: Duration,
+) -> Duration {
+	// 1 extra block for transaction to reach the pool && 1 for relayer to awake after it is mined
+	mortality_period
+		.map(|mortality_period| average_block_interval.saturating_mul(mortality_period + 1 + 1))
+		.unwrap_or(default_stall_timeout)
+}

pezbridges/relays/client-bizinikiwi/src/metrics/float_storage_value.rs

@@ -0,0 +1,140 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{Chain, Client, Error as BizinikiwiError};
+
+use async_std::sync::{Arc, RwLock};
+use async_trait::async_trait;
+use codec::Decode;
+use num_traits::One;
+use relay_utils::metrics::{
+	metric_name, register, F64SharedRef, Gauge, Metric, PrometheusError, Registry,
+	StandaloneMetric, F64,
+};
+use pezsp_core::storage::{StorageData, StorageKey};
+use pezsp_runtime::{traits::UniqueSaturatedInto, FixedPointNumber, FixedU128};
+use std::{marker::PhantomData, time::Duration};
+
+/// Storage value update interval (in blocks).
+const UPDATE_INTERVAL_IN_BLOCKS: u32 = 5;
+
+/// Fied-point storage value and the way it is decoded from the raw storage value.
+pub trait FloatStorageValue: 'static + Clone + Send + Sync {
+	/// Type of the value.
+	type Value: FixedPointNumber;
+	/// Try to decode value from the raw storage value.
+	fn decode(
+		&self,
+		maybe_raw_value: Option<StorageData>,
+	) -> Result<Option<Self::Value>, BizinikiwiError>;
+}
+
+/// Implementation of `FloatStorageValue` that expects encoded `FixedU128` value and returns `1` if
+/// value is missing from the storage.
+#[derive(Clone, Debug, Default)]
+pub struct FixedU128OrOne;
+
+impl FloatStorageValue for FixedU128OrOne {
+	type Value = FixedU128;
+
+	fn decode(
+		&self,
+		maybe_raw_value: Option<StorageData>,
+	) -> Result<Option<Self::Value>, BizinikiwiError> {
+		maybe_raw_value
+			.map(|raw_value| {
+				FixedU128::decode(&mut &raw_value.0[..])
+					.map_err(BizinikiwiError::ResponseParseFailed)
+					.map(Some)
+			})
+			.unwrap_or_else(|| Ok(Some(FixedU128::one())))
+	}
+}
+
+/// Metric that represents fixed-point runtime storage value as float gauge.
+#[derive(Clone, Debug)]
+pub struct FloatStorageValueMetric<C, Clnt, V> {
+	value_converter: V,
+	client: Clnt,
+	storage_key: StorageKey,
+	metric: Gauge<F64>,
+	shared_value_ref: F64SharedRef,
+	_phantom: PhantomData<(C, V)>,
+}
+
+impl<C, Clnt, V> FloatStorageValueMetric<C, Clnt, V> {
+	/// Create new metric.
+	pub fn new(
+		value_converter: V,
+		client: Clnt,
+		storage_key: StorageKey,
+		name: String,
+		help: String,
+	) -> Result<Self, PrometheusError> {
+		let shared_value_ref = Arc::new(RwLock::new(None));
+		Ok(FloatStorageValueMetric {
+			value_converter,
+			client,
+			storage_key,
+			metric: Gauge::new(metric_name(None, &name), help)?,
+			shared_value_ref,
+			_phantom: Default::default(),
+		})
+	}
+
+	/// Get shared reference to metric value.
+	pub fn shared_value_ref(&self) -> F64SharedRef {
+		self.shared_value_ref.clone()
+	}
+}
+
+impl<C: Chain, Clnt: Client<C>, V: FloatStorageValue> Metric
+	for FloatStorageValueMetric<C, Clnt, V>
+{
+	fn register(&self, registry: &Registry) -> Result<(), PrometheusError> {
+		register(self.metric.clone(), registry).map(drop)
+	}
+}
+
+#[async_trait]
+impl<C: Chain, Clnt: Client<C>, V: FloatStorageValue> StandaloneMetric
+	for FloatStorageValueMetric<C, Clnt, V>
+{
+	fn update_interval(&self) -> Duration {
+		C::AVERAGE_BLOCK_INTERVAL * UPDATE_INTERVAL_IN_BLOCKS
+	}
+
+	async fn update(&self) {
+		let value = async move {
+			let best_header_hash = self.client.best_header_hash().await?;
+			let maybe_storage_value = self
+				.client
+				.raw_storage_value(best_header_hash, self.storage_key.clone())
+				.await?;
+			self.value_converter.decode(maybe_storage_value).map(|maybe_fixed_point_value| {
+				maybe_fixed_point_value.map(|fixed_point_value| {
+					fixed_point_value.into_inner().unique_saturated_into() as f64 /
+						V::Value::DIV.unique_saturated_into() as f64
+				})
+			})
+		}
+		.await
+		.map_err(|e| e.to_string());
+
+		relay_utils::metrics::set_gauge_value(&self.metric, value.clone());
+		*self.shared_value_ref.write().await = value.ok().and_then(|x| x);
+	}
+}

pezbridges/relays/client-bizinikiwi/src/metrics/mod.rs

@@ -0,0 +1,21 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Contains several Bizinikiwi-specific metrics that may be exposed by relay.
+
+pub use float_storage_value::{FixedU128OrOne, FloatStorageValue, FloatStorageValueMetric};
+
+mod float_storage_value;

pezbridges/relays/client-bizinikiwi/src/sync_header.rs

@@ -0,0 +1,61 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use bp_header_pez_chain::ConsensusLogReader;
+use pez_finality_relay::SourceHeader as FinalitySourceHeader;
+use pezsp_runtime::traits::Header as HeaderT;
+
+/// Generic wrapper for `pezsp_runtime::traits::Header` based headers, that
+/// implements `pez_finality_relay::SourceHeader` and may be used in headers sync directly.
+#[derive(Clone, Debug, PartialEq)]
+pub struct SyncHeader<Header>(Header);
+
+impl<Header> SyncHeader<Header> {
+	/// Extracts wrapped header from self.
+	pub fn into_inner(self) -> Header {
+		self.0
+	}
+}
+
+impl<Header> std::ops::Deref for SyncHeader<Header> {
+	type Target = Header;
+
+	fn deref(&self) -> &Self::Target {
+		&self.0
+	}
+}
+
+impl<Header> From<Header> for SyncHeader<Header> {
+	fn from(header: Header) -> Self {
+		Self(header)
+	}
+}
+
+impl<Header: HeaderT, R: ConsensusLogReader> FinalitySourceHeader<Header::Hash, Header::Number, R>
+	for SyncHeader<Header>
+{
+	fn hash(&self) -> Header::Hash {
+		self.0.hash()
+	}
+
+	fn number(&self) -> Header::Number {
+		*self.0.number()
+	}
+
+	fn is_mandatory(&self) -> bool {
+		R::schedules_authorities_change(self.digest())
+	}
+}

pezbridges/relays/client-bizinikiwi/src/test_chain.rs

@@ -0,0 +1,200 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Pezpallet provides a set of guard functions that are running in background threads
+//! and are aborting process if some condition fails.
+
+//! Test chain implementation to use in tests.
+
+#![cfg(any(feature = "test-helpers", test))]
+
+use crate::{
+	Chain, ChainWithBalances, ChainWithMessages, ChainWithRewards, ChainWithTransactions,
+	Error as BizinikiwiError, SignParam, UnsignedTransaction,
+};
+use bp_messages::{ChainWithMessages as ChainWithMessagesBase, MessageNonce};
+use pezbp_runtime::ChainId;
+use codec::{Decode, DecodeWithMemTracking, Encode, MaxEncodedLen};
+use pezframe_support::{pezsp_runtime::StateVersion, weights::Weight};
+use scale_info::TypeInfo;
+use std::time::Duration;
+
+/// Chain that may be used in tests.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct TestChain;
+
+impl pezbp_runtime::Chain for TestChain {
+	const ID: ChainId = *b"test";
+
+	type BlockNumber = u32;
+	type Hash = pezsp_core::H256;
+	type Hasher = pezsp_runtime::traits::BlakeTwo256;
+	type Header = pezsp_runtime::generic::Header<u32, pezsp_runtime::traits::BlakeTwo256>;
+
+	type AccountId = u32;
+	type Balance = u32;
+	type Nonce = u32;
+	type Signature = pezsp_runtime::testing::TestSignature;
+
+	const STATE_VERSION: StateVersion = StateVersion::V1;
+
+	fn max_extrinsic_size() -> u32 {
+		100000
+	}
+
+	fn max_extrinsic_weight() -> Weight {
+		unreachable!()
+	}
+}
+
+impl Chain for TestChain {
+	const NAME: &'static str = "Test";
+	const BEST_FINALIZED_HEADER_ID_METHOD: &'static str = "TestMethod";
+	const FREE_HEADERS_INTERVAL_METHOD: &'static str = "TestMethod";
+	const AVERAGE_BLOCK_INTERVAL: Duration = Duration::from_millis(0);
+
+	type SignedBlock = pezsp_runtime::generic::SignedBlock<
+		pezsp_runtime::generic::Block<Self::Header, pezsp_runtime::OpaqueExtrinsic>,
+	>;
+	type Call = TestRuntimeCall;
+}
+
+impl ChainWithBalances for TestChain {
+	fn account_info_storage_key(_account_id: &u32) -> pezsp_core::storage::StorageKey {
+		unreachable!()
+	}
+}
+
+/// Reward type for the test chain.
+#[derive(
+	Clone,
+	Copy,
+	Debug,
+	Decode,
+	DecodeWithMemTracking,
+	Encode,
+	Eq,
+	MaxEncodedLen,
+	PartialEq,
+	TypeInfo,
+)]
+pub enum ChainReward {
+	/// Reward 1 type.
+	Reward1,
+}
+
+impl ChainWithRewards for TestChain {
+	const WITH_CHAIN_RELAYERS_PALLET_NAME: Option<&'static str> = None;
+	type RewardBalance = u128;
+	type Reward = ChainReward;
+
+	fn account_reward_storage_key(
+		_account_id: &Self::AccountId,
+		_reward: impl Into<Self::Reward>,
+	) -> pezsp_core::storage::StorageKey {
+		unreachable!()
+	}
+}
+
+impl ChainWithMessagesBase for TestChain {
+	const WITH_CHAIN_MESSAGES_PALLET_NAME: &'static str = "Test";
+	const MAX_UNREWARDED_RELAYERS_IN_CONFIRMATION_TX: MessageNonce = 0;
+	const MAX_UNCONFIRMED_MESSAGES_IN_CONFIRMATION_TX: MessageNonce = 0;
+}
+
+impl ChainWithMessages for TestChain {
+	const TO_CHAIN_MESSAGE_DETAILS_METHOD: &'static str = "TestMessagesDetailsMethod";
+	const FROM_CHAIN_MESSAGE_DETAILS_METHOD: &'static str = "TestFromMessagesDetailsMethod";
+}
+
+impl ChainWithTransactions for TestChain {
+	type AccountKeyPair = pezsp_core::sr25519::Pair;
+	type SignedTransaction = bp_pezkuwi_core::UncheckedExtrinsic<
+		TestRuntimeCall,
+		bp_pezkuwi_core::SuffixedCommonTransactionExtension<(
+			pezbp_runtime::extensions::BridgeRejectObsoleteHeadersAndMessages,
+			pezbp_runtime::extensions::RefundBridgedTeyrchainMessagesSchema,
+		)>,
+	>;
+
+	fn sign_transaction(
+		_param: SignParam<Self>,
+		_unsigned: UnsignedTransaction<Self>,
+	) -> Result<Self::SignedTransaction, BizinikiwiError> {
+		unreachable!()
+	}
+}
+
+/// Dummy runtime call.
+#[derive(Decode, Encode, Clone, Debug, PartialEq)]
+pub enum TestRuntimeCall {
+	/// Dummy call.
+	#[codec(index = 0)]
+	Dummy,
+}
+
+/// Primitives-level teyrchain that may be used in tests.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct TestTeyrchainBase;
+
+impl pezbp_runtime::Chain for TestTeyrchainBase {
+	const ID: ChainId = *b"tstp";
+
+	type BlockNumber = u32;
+	type Hash = pezsp_core::H256;
+	type Hasher = pezsp_runtime::traits::BlakeTwo256;
+	type Header = pezsp_runtime::generic::Header<u32, pezsp_runtime::traits::BlakeTwo256>;
+
+	type AccountId = u32;
+	type Balance = u32;
+	type Nonce = u32;
+	type Signature = pezsp_runtime::testing::TestSignature;
+
+	const STATE_VERSION: StateVersion = StateVersion::V1;
+
+	fn max_extrinsic_size() -> u32 {
+		unreachable!()
+	}
+
+	fn max_extrinsic_weight() -> Weight {
+		unreachable!()
+	}
+}
+
+impl pezbp_runtime::Teyrchain for TestTeyrchainBase {
+	const TEYRCHAIN_ID: u32 = 1000;
+	const MAX_HEADER_SIZE: u32 = 1_024;
+}
+
+/// Teyrchain that may be used in tests.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct TestTeyrchain;
+
+impl pezbp_runtime::UnderlyingChainProvider for TestTeyrchain {
+	type Chain = TestTeyrchainBase;
+}
+
+impl Chain for TestTeyrchain {
+	const NAME: &'static str = "TestTeyrchain";
+	const BEST_FINALIZED_HEADER_ID_METHOD: &'static str = "TestTeyrchainMethod";
+	const FREE_HEADERS_INTERVAL_METHOD: &'static str = "TestTeyrchainMethod";
+	const AVERAGE_BLOCK_INTERVAL: Duration = Duration::from_millis(0);
+
+	type SignedBlock = pezsp_runtime::generic::SignedBlock<
+		pezsp_runtime::generic::Block<Self::Header, pezsp_runtime::OpaqueExtrinsic>,
+	>;
+	type Call = TestRuntimeCall;
+}

pezbridges/relays/client-bizinikiwi/src/transaction_tracker.rs

@@ -0,0 +1,473 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Helper for tracking transaction invalidation events.
+
+use crate::{Chain, Error, HashOf, HeaderIdOf, Subscription, TransactionStatusOf};
+
+use async_trait::async_trait;
+use futures::{future::Either, Future, FutureExt, Stream, StreamExt};
+use relay_utils::{HeaderId, TrackedTransactionStatus};
+use pezsp_runtime::traits::Header as _;
+use std::time::Duration;
+
+/// Transaction tracker environment.
+#[async_trait]
+pub trait Environment<C: Chain>: Send + Sync {
+	/// Returns header id by its hash.
+	async fn header_id_by_hash(&self, hash: HashOf<C>) -> Result<HeaderIdOf<C>, Error>;
+}
+
+// TODO (https://github.com/pezkuwichain/kurdistan-sdk/issues/84): remove `Environment` trait
+// after test client is implemented
+#[async_trait]
+impl<C: Chain, T: crate::client::Client<C>> Environment<C> for T {
+	async fn header_id_by_hash(&self, hash: HashOf<C>) -> Result<HeaderIdOf<C>, Error> {
+		self.header_by_hash(hash).await.map(|h| HeaderId(*h.number(), hash))
+	}
+}
+
+/// Bizinikiwi transaction tracker implementation.
+///
+/// Bizinikiwi node provides RPC API to submit and watch for transaction events. This way
+/// we may know when transaction is included into block, finalized or rejected. There are
+/// some edge cases, when we can't fully trust this mechanism - e.g. transaction may broadcasted
+/// and then dropped out of node transaction pool (some other cases are also possible - node
+/// restarts, connection lost, ...). Then we can't know for sure - what is currently happening
+/// with our transaction. Is the transaction really lost? Is it still alive on the chain network?
+///
+/// We have several options to handle such cases:
+///
+/// 1) hope that the transaction is still alive and wait for its mining until it is spoiled;
+///
+/// 2) assume that the transaction is lost and resubmit another transaction instantly;
+///
+/// 3) wait for some time (if transaction is mortal - then until block where it dies; if it is
+///    immortal - then for some time that we assume is long enough to mine it) and assume that it is
+///    lost.
+///
+/// This struct implements third option as it seems to be the most optimal.
+pub struct TransactionTracker<C: Chain, E> {
+	environment: E,
+	transaction_hash: HashOf<C>,
+	stall_timeout: Duration,
+	subscription: Subscription<TransactionStatusOf<C>>,
+}
+
+impl<C: Chain, E: Environment<C>> TransactionTracker<C, E> {
+	/// Create transaction tracker.
+	pub fn new(
+		environment: E,
+		stall_timeout: Duration,
+		transaction_hash: HashOf<C>,
+		subscription: Subscription<TransactionStatusOf<C>>,
+	) -> Self {
+		Self { environment, stall_timeout, transaction_hash, subscription }
+	}
+
+	// TODO (https://github.com/pezkuwichain/kurdistan-sdk/issues/84): remove me after
+	// test client is implemented
+	/// Converts self into tracker with different environment.
+	pub fn switch_environment<NewE: Environment<C>>(
+		self,
+		environment: NewE,
+	) -> TransactionTracker<C, NewE> {
+		TransactionTracker {
+			environment,
+			stall_timeout: self.stall_timeout,
+			transaction_hash: self.transaction_hash,
+			subscription: self.subscription,
+		}
+	}
+
+	/// Wait for final transaction status and return it along with last known internal invalidation
+	/// status.
+	async fn do_wait(
+		self,
+		wait_for_stall_timeout: impl Future<Output = ()>,
+		wait_for_stall_timeout_rest: impl Future<Output = ()>,
+	) -> (TrackedTransactionStatus<HeaderIdOf<C>>, Option<InvalidationStatus<HeaderIdOf<C>>>) {
+		// sometimes we want to wait for the rest of the stall timeout even if
+		// `wait_for_invalidation` has been "select"ed first => it is shared
+		let wait_for_invalidation = watch_transaction_status::<_, C, _>(
+			self.environment,
+			self.transaction_hash,
+			self.subscription,
+		);
+		futures::pin_mut!(wait_for_stall_timeout, wait_for_invalidation);
+
+		match futures::future::select(wait_for_stall_timeout, wait_for_invalidation).await {
+			Either::Left((_, _)) => {
+				tracing::trace!(
+					target: "bridge",
+					node=%C::NAME,
+					transaction=?self.transaction_hash,
+					"Transaction is considered lost after timeout (no status response from the node)"
+				);
+
+				(TrackedTransactionStatus::Lost, None)
+			},
+			Either::Right((invalidation_status, _)) => match invalidation_status {
+				InvalidationStatus::Finalized(at_block) =>
+					(TrackedTransactionStatus::Finalized(at_block), Some(invalidation_status)),
+				InvalidationStatus::Invalid =>
+					(TrackedTransactionStatus::Lost, Some(invalidation_status)),
+				InvalidationStatus::Lost => {
+					// wait for the rest of stall timeout - this way we'll be sure that the
+					// transaction is actually dead if it has been crafted properly
+					wait_for_stall_timeout_rest.await;
+					// if someone is still watching for our transaction, then we're reporting
+					// an error here (which is treated as "transaction lost")
+					tracing::trace!(
+						target: "bridge",
+						node=%C::NAME,
+						transaction=?self.transaction_hash,
+						"Transaction is considered lost after timeout"
+					);
+
+					(TrackedTransactionStatus::Lost, Some(invalidation_status))
+				},
+			},
+		}
+	}
+}
+
+#[async_trait]
+impl<C: Chain, E: Environment<C>> relay_utils::TransactionTracker for TransactionTracker<C, E> {
+	type HeaderId = HeaderIdOf<C>;
+
+	async fn wait(self) -> TrackedTransactionStatus<HeaderIdOf<C>> {
+		let wait_for_stall_timeout = async_std::task::sleep(self.stall_timeout).shared();
+		let wait_for_stall_timeout_rest = wait_for_stall_timeout.clone();
+		self.do_wait(wait_for_stall_timeout, wait_for_stall_timeout_rest).await.0
+	}
+}
+
+/// Transaction invalidation status.
+///
+/// Note that in places where the `TransactionTracker` is used, the finalization event will be
+/// ignored - relay loops are detecting the mining/finalization using their own
+/// techniques. That's why we're using `InvalidationStatus` here.
+#[derive(Debug, PartialEq)]
+enum InvalidationStatus<BlockId> {
+	/// Transaction has been included into block and finalized at given block.
+	Finalized(BlockId),
+	/// Transaction has been invalidated.
+	Invalid,
+	/// We have lost track of transaction status.
+	Lost,
+}
+
+/// Watch for transaction status until transaction is finalized or we lose track of its status.
+async fn watch_transaction_status<
+	E: Environment<C>,
+	C: Chain,
+	S: Stream<Item = TransactionStatusOf<C>>,
+>(
+	environment: E,
+	transaction_hash: HashOf<C>,
+	subscription: S,
+) -> InvalidationStatus<HeaderIdOf<C>> {
+	futures::pin_mut!(subscription);
+
+	loop {
+		match subscription.next().await {
+			Some(TransactionStatusOf::<C>::Finalized((block_hash, _))) => {
+				// the only "successful" outcome of this method is when the block with transaction
+				// has been finalized
+				tracing::trace!(
+					target: "bridge",
+					node=%C::NAME,
+					transaction=?transaction_hash,
+					block=?block_hash,
+					"Transaction has been finalized"
+				);
+
+				let header_id = match environment.header_id_by_hash(block_hash).await {
+					Ok(header_id) => header_id,
+					Err(e) => {
+						tracing::error!(
+							target: "bridge",
+							error=?e,
+							node=%C::NAME,
+							transaction=?transaction_hash,
+							block=?block_hash,
+							"Failed to read header when watching for transaction",
+						);
+						// that's the best option we have here
+						return InvalidationStatus::Lost;
+					},
+				};
+				return InvalidationStatus::Finalized(header_id);
+			},
+			Some(TransactionStatusOf::<C>::Invalid) => {
+				// if node says that the transaction is invalid, there are still chances that
+				// it is not actually invalid - e.g. if the block where transaction has been
+				// revalidated is retracted and transaction (at some other node pool) becomes
+				// valid again on other fork. But let's assume that the chances of this event
+				// are almost zero - there's a lot of things that must happen for this to be the
+				// case.
+				tracing::trace!(
+					target: "bridge",
+					node=%C::NAME,
+					transaction=?transaction_hash,
+					"Transaction has been invalidated"
+				);
+				return InvalidationStatus::Invalid;
+			},
+			Some(TransactionStatusOf::<C>::Future) |
+			Some(TransactionStatusOf::<C>::Ready) |
+			Some(TransactionStatusOf::<C>::Broadcast(_)) => {
+				// nothing important (for us) has happened
+			},
+			Some(TransactionStatusOf::<C>::InBlock(block_hash)) => {
+				// TODO: read matching system event (ExtrinsicSuccess or ExtrinsicFailed), log it
+				// here and use it later (on finality) for reporting invalid transaction
+				// https://github.com/pezkuwichain/kurdistan-sdk/issues/79
+				tracing::trace!(
+					target: "bridge",
+					node=%C::NAME,
+					transaction=?transaction_hash,
+					block=?block_hash,
+					"Transaction has been included"
+				);
+			},
+			Some(TransactionStatusOf::<C>::Retracted(block_hash)) => {
+				tracing::trace!(
+					target: "bridge",
+					node=%C::NAME,
+					transaction=?transaction_hash,
+					block=?block_hash,
+					"Transaction has been retracted"
+				);
+			},
+			Some(TransactionStatusOf::<C>::FinalityTimeout(block_hash)) => {
+				// finality is lagging? let's wait a bit more and report a stall
+				tracing::trace!(
+					target: "bridge",
+					node=%C::NAME,
+					transaction=?transaction_hash,
+					block=?block_hash,
+					"Transaction has not been finalized for too long"
+				);
+				return InvalidationStatus::Lost;
+			},
+			Some(TransactionStatusOf::<C>::Usurped(new_transaction_hash)) => {
+				// this may be result of our transaction resubmitter work or some manual
+				// intervention. In both cases - let's start stall timeout, because the meaning
+				// of transaction may have changed
+				tracing::trace!(
+					target: "bridge",
+					node=%C::NAME,
+					transaction=?transaction_hash,
+					new_transaction=?new_transaction_hash,
+					"Transaction has been usurped"
+				);
+				return InvalidationStatus::Lost;
+			},
+			Some(TransactionStatusOf::<C>::Dropped) => {
+				// the transaction has been removed from the pool because of its limits. Let's wait
+				// a bit and report a stall
+				tracing::trace!(
+					target: "bridge",
+					node=%C::NAME,
+					transaction=?transaction_hash,
+					"Transaction has been dropped from the pool"
+				);
+				return InvalidationStatus::Lost;
+			},
+			None => {
+				// the status of transaction is unknown to us (the subscription has been closed?).
+				// Let's wait a bit and report a stall
+				return InvalidationStatus::Lost;
+			},
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::{test_chain::TestChain, StreamDescription};
+	use futures::{FutureExt, SinkExt};
+	use pezsc_transaction_pool_api::TransactionStatus;
+
+	struct TestEnvironment(Result<HeaderIdOf<TestChain>, Error>);
+
+	#[async_trait]
+	impl Environment<TestChain> for TestEnvironment {
+		async fn header_id_by_hash(
+			&self,
+			_hash: HashOf<TestChain>,
+		) -> Result<HeaderIdOf<TestChain>, Error> {
+			self.0.as_ref().map_err(|_| Error::BridgePalletIsNotInitialized).cloned()
+		}
+	}
+
+	async fn on_transaction_status(
+		status: TransactionStatus<HashOf<TestChain>, HashOf<TestChain>>,
+	) -> Option<(
+		TrackedTransactionStatus<HeaderIdOf<TestChain>>,
+		InvalidationStatus<HeaderIdOf<TestChain>>,
+	)> {
+		let (mut sender, receiver) = futures::channel::mpsc::channel(1);
+		let tx_tracker = TransactionTracker::<TestChain, TestEnvironment>::new(
+			TestEnvironment(Ok(HeaderId(0, Default::default()))),
+			Duration::from_secs(0),
+			Default::default(),
+			Subscription::new_forwarded(
+				StreamDescription::new("test".into(), "test".into()),
+				receiver,
+			),
+		);
+
+		// we can't do `.now_or_never()` on `do_wait()` call, because `Subscription` has its own
+		// background thread, which may cause additional async task switches => let's leave some
+		// relatively small timeout here
+		let wait_for_stall_timeout = async_std::task::sleep(std::time::Duration::from_millis(100));
+		let wait_for_stall_timeout_rest = futures::future::ready(());
+		sender.send(Ok(status)).await.unwrap();
+
+		let (ts, is) =
+			tx_tracker.do_wait(wait_for_stall_timeout, wait_for_stall_timeout_rest).await;
+		is.map(|is| (ts, is))
+	}
+
+	#[async_std::test]
+	async fn returns_finalized_on_finalized() {
+		assert_eq!(
+			on_transaction_status(TransactionStatus::Finalized(Default::default())).await,
+			Some((
+				TrackedTransactionStatus::Finalized(Default::default()),
+				InvalidationStatus::Finalized(Default::default())
+			)),
+		);
+	}
+
+	#[async_std::test]
+	async fn returns_lost_on_finalized_and_environment_error() {
+		assert_eq!(
+			watch_transaction_status::<_, TestChain, _>(
+				TestEnvironment(Err(Error::BridgePalletIsNotInitialized)),
+				Default::default(),
+				futures::stream::iter([TransactionStatus::Finalized(Default::default())])
+			)
+			.now_or_never(),
+			Some(InvalidationStatus::Lost),
+		);
+	}
+
+	#[async_std::test]
+	async fn returns_invalid_on_invalid() {
+		assert_eq!(
+			on_transaction_status(TransactionStatus::Invalid).await,
+			Some((TrackedTransactionStatus::Lost, InvalidationStatus::Invalid)),
+		);
+	}
+
+	#[async_std::test]
+	async fn waits_on_future() {
+		assert_eq!(on_transaction_status(TransactionStatus::Future).await, None,);
+	}
+
+	#[async_std::test]
+	async fn waits_on_ready() {
+		assert_eq!(on_transaction_status(TransactionStatus::Ready).await, None,);
+	}
+
+	#[async_std::test]
+	async fn waits_on_broadcast() {
+		assert_eq!(
+			on_transaction_status(TransactionStatus::Broadcast(Default::default())).await,
+			None,
+		);
+	}
+
+	#[async_std::test]
+	async fn waits_on_in_block() {
+		assert_eq!(
+			on_transaction_status(TransactionStatus::InBlock(Default::default())).await,
+			None,
+		);
+	}
+
+	#[async_std::test]
+	async fn waits_on_retracted() {
+		assert_eq!(
+			on_transaction_status(TransactionStatus::Retracted(Default::default())).await,
+			None,
+		);
+	}
+
+	#[async_std::test]
+	async fn lost_on_finality_timeout() {
+		assert_eq!(
+			on_transaction_status(TransactionStatus::FinalityTimeout(Default::default())).await,
+			Some((TrackedTransactionStatus::Lost, InvalidationStatus::Lost)),
+		);
+	}
+
+	#[async_std::test]
+	async fn lost_on_usurped() {
+		assert_eq!(
+			on_transaction_status(TransactionStatus::Usurped(Default::default())).await,
+			Some((TrackedTransactionStatus::Lost, InvalidationStatus::Lost)),
+		);
+	}
+
+	#[async_std::test]
+	async fn lost_on_dropped() {
+		assert_eq!(
+			on_transaction_status(TransactionStatus::Dropped).await,
+			Some((TrackedTransactionStatus::Lost, InvalidationStatus::Lost)),
+		);
+	}
+
+	#[async_std::test]
+	async fn lost_on_subscription_error() {
+		assert_eq!(
+			watch_transaction_status::<_, TestChain, _>(
+				TestEnvironment(Ok(HeaderId(0, Default::default()))),
+				Default::default(),
+				futures::stream::iter([])
+			)
+			.now_or_never(),
+			Some(InvalidationStatus::Lost),
+		);
+	}
+
+	#[async_std::test]
+	async fn lost_on_timeout_when_waiting_for_invalidation_status() {
+		let (_sender, receiver) = futures::channel::mpsc::channel(1);
+		let tx_tracker = TransactionTracker::<TestChain, TestEnvironment>::new(
+			TestEnvironment(Ok(HeaderId(0, Default::default()))),
+			Duration::from_secs(0),
+			Default::default(),
+			Subscription::new_forwarded(
+				StreamDescription::new("test".into(), "test".into()),
+				receiver,
+			),
+		);
+
+		let wait_for_stall_timeout = futures::future::ready(()).shared();
+		let wait_for_stall_timeout_rest = wait_for_stall_timeout.clone();
+		let wait_result = tx_tracker
+			.do_wait(wait_for_stall_timeout, wait_for_stall_timeout_rest)
+			.now_or_never();
+
+		assert_eq!(wait_result, Some((TrackedTransactionStatus::Lost, None)));
+	}
+}

pezbridges/relays/equivocation/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "pez-equivocation-detector"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+repository.workspace = true
+description = "Equivocation detector"
+publish = false
+documentation = "https://docs.rs/pez-equivocation-detector"
+homepage = { workspace = true }
+
+[lints]
+workspace = true
+
+[dependencies]
+async-std = { features = ["attributes"], workspace = true }
+async-trait = { workspace = true }
+bp-header-pez-chain = { workspace = true, default-features = true }
+pez-finality-relay = { workspace = true }
+futures = { workspace = true }
+num-traits = { workspace = true, default-features = true }
+relay-utils = { workspace = true }
+tracing = { workspace = true }
+
+[features]
+runtime-benchmarks = [
+  "bp-header-pez-chain/runtime-benchmarks",
+  "pez-finality-relay/runtime-benchmarks",
+  "relay-utils/runtime-benchmarks",
+]

pezbridges/relays/equivocation/src/block_checker.rs

@@ -0,0 +1,476 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{
+	handle_client_error, reporter::EquivocationsReporter, EquivocationDetectionPipeline,
+	EquivocationReportingContext, HeaderFinalityInfo, SourceClient, TargetClient,
+};
+
+use bp_header_pez_chain::{FinalityProof, FindEquivocations as FindEquivocationsT};
+use pez_finality_relay::FinalityProofsBuf;
+use futures::future::{BoxFuture, FutureExt};
+use num_traits::Saturating;
+
+/// First step in the block checking state machine.
+///
+/// Getting the finality info associated to the source headers synced with the target chain
+/// at the specified block.
+#[cfg_attr(test, derive(Debug, PartialEq))]
+pub struct ReadSyncedHeaders<P: EquivocationDetectionPipeline> {
+	pub target_block_num: P::TargetNumber,
+}
+
+impl<P: EquivocationDetectionPipeline> ReadSyncedHeaders<P> {
+	pub async fn next<TC: TargetClient<P>>(
+		self,
+		target_client: &mut TC,
+	) -> Result<ReadContext<P>, Self> {
+		match target_client.synced_headers_finality_info(self.target_block_num).await {
+			Ok(synced_headers) =>
+				Ok(ReadContext { target_block_num: self.target_block_num, synced_headers }),
+			Err(e) => {
+				tracing::error!(
+					target: "bridge",
+					error=?e,
+					source=%P::SOURCE_NAME,
+					target=%P::TARGET_NAME,
+					block=%self.target_block_num,
+					"Could not get headers synced at block"
+				);
+
+				// Reconnect target client in case of a connection error.
+				handle_client_error(target_client, e).await;
+
+				Err(self)
+			},
+		}
+	}
+}
+
+/// Second step in the block checking state machine.
+///
+/// Reading the equivocation reporting context from the target chain.
+#[cfg_attr(test, derive(Debug))]
+pub struct ReadContext<P: EquivocationDetectionPipeline> {
+	target_block_num: P::TargetNumber,
+	synced_headers: Vec<HeaderFinalityInfo<P>>,
+}
+
+impl<P: EquivocationDetectionPipeline> ReadContext<P> {
+	pub async fn next<TC: TargetClient<P>>(
+		self,
+		target_client: &mut TC,
+	) -> Result<Option<FindEquivocations<P>>, Self> {
+		match EquivocationReportingContext::try_read_from_target::<TC>(
+			target_client,
+			self.target_block_num.saturating_sub(1.into()),
+		)
+		.await
+		{
+			Ok(Some(context)) => Ok(Some(FindEquivocations {
+				target_block_num: self.target_block_num,
+				synced_headers: self.synced_headers,
+				context,
+			})),
+			Ok(None) => Ok(None),
+			Err(e) => {
+				tracing::error!(
+					target: "bridge",
+					error=?e,
+					source=%P::SOURCE_NAME,
+					target=%P::TARGET_NAME,
+					block=%self.target_block_num.saturating_sub(1.into()),
+					"Could not read `EquivocationReportingContext` at block",
+				);
+
+				// Reconnect target client in case of a connection error.
+				handle_client_error(target_client, e).await;
+
+				Err(self)
+			},
+		}
+	}
+}
+
+/// Third step in the block checking state machine.
+///
+/// Searching for equivocations in the source headers synced with the target chain.
+#[cfg_attr(test, derive(Debug))]
+pub struct FindEquivocations<P: EquivocationDetectionPipeline> {
+	target_block_num: P::TargetNumber,
+	synced_headers: Vec<HeaderFinalityInfo<P>>,
+	context: EquivocationReportingContext<P>,
+}
+
+impl<P: EquivocationDetectionPipeline> FindEquivocations<P> {
+	pub fn next(
+		mut self,
+		finality_proofs_buf: &mut FinalityProofsBuf<P>,
+	) -> Vec<ReportEquivocations<P>> {
+		let mut result = vec![];
+		for synced_header in self.synced_headers {
+			match P::EquivocationsFinder::find_equivocations(
+				&self.context.synced_verification_context,
+				&synced_header.finality_proof,
+				finality_proofs_buf.buf().as_slice(),
+			) {
+				Ok(equivocations) =>
+					if !equivocations.is_empty() {
+						result.push(ReportEquivocations {
+							source_block_hash: self.context.synced_header_hash,
+							equivocations,
+						})
+					},
+				Err(e) => {
+					tracing::error!(
+						target: "bridge",
+						error=?e,
+						source_header=?synced_header.finality_proof.target_header_hash(),
+						block=%self.target_block_num,
+						"Could not search for equivocations in the finality proof \
+						for source header synced at target block"
+					);
+				},
+			};
+
+			finality_proofs_buf.prune(synced_header.finality_proof.target_header_number(), None);
+			self.context.update(synced_header);
+		}
+
+		result
+	}
+}
+
+/// Fourth step in the block checking state machine.
+///
+/// Reporting the detected equivocations (if any).
+#[cfg_attr(test, derive(Debug))]
+pub struct ReportEquivocations<P: EquivocationDetectionPipeline> {
+	source_block_hash: P::Hash,
+	equivocations: Vec<P::EquivocationProof>,
+}
+
+impl<P: EquivocationDetectionPipeline> ReportEquivocations<P> {
+	pub async fn next<SC: SourceClient<P>>(
+		mut self,
+		source_client: &mut SC,
+		reporter: &mut EquivocationsReporter<'_, P, SC>,
+	) -> Result<(), Self> {
+		let mut unprocessed_equivocations = vec![];
+		for equivocation in self.equivocations {
+			match reporter
+				.submit_report(source_client, self.source_block_hash, equivocation.clone())
+				.await
+			{
+				Ok(_) => {},
+				Err(e) => {
+					tracing::error!(
+						target: "bridge",
+						error=?e,
+						source=%P::SOURCE_NAME,
+						?equivocation,
+						"Could not submit equivocation report"
+					);
+
+					// Mark the equivocation as unprocessed
+					unprocessed_equivocations.push(equivocation);
+					// Reconnect source client in case of a connection error.
+					handle_client_error(source_client, e).await;
+				},
+			}
+		}
+
+		self.equivocations = unprocessed_equivocations;
+		if !self.equivocations.is_empty() {
+			return Err(self);
+		}
+
+		Ok(())
+	}
+}
+
+/// Block checking state machine.
+#[cfg_attr(test, derive(Debug))]
+pub enum BlockChecker<P: EquivocationDetectionPipeline> {
+	ReadSyncedHeaders(ReadSyncedHeaders<P>),
+	ReadContext(ReadContext<P>),
+	ReportEquivocations(Vec<ReportEquivocations<P>>),
+}
+
+impl<P: EquivocationDetectionPipeline> BlockChecker<P> {
+	pub fn new(target_block_num: P::TargetNumber) -> Self {
+		Self::ReadSyncedHeaders(ReadSyncedHeaders { target_block_num })
+	}
+
+	pub fn run<'a, SC: SourceClient<P>, TC: TargetClient<P>>(
+		self,
+		source_client: &'a mut SC,
+		target_client: &'a mut TC,
+		finality_proofs_buf: &'a mut FinalityProofsBuf<P>,
+		reporter: &'a mut EquivocationsReporter<P, SC>,
+	) -> BoxFuture<'a, Result<(), Self>> {
+		async move {
+			match self {
+				Self::ReadSyncedHeaders(state) => {
+					let read_context =
+						state.next(target_client).await.map_err(Self::ReadSyncedHeaders)?;
+					Self::ReadContext(read_context)
+						.run(source_client, target_client, finality_proofs_buf, reporter)
+						.await
+				},
+				Self::ReadContext(state) => {
+					let maybe_find_equivocations =
+						state.next(target_client).await.map_err(Self::ReadContext)?;
+					let find_equivocations = match maybe_find_equivocations {
+						Some(find_equivocations) => find_equivocations,
+						None => return Ok(()),
+					};
+					Self::ReportEquivocations(find_equivocations.next(finality_proofs_buf))
+						.run(source_client, target_client, finality_proofs_buf, reporter)
+						.await
+				},
+				Self::ReportEquivocations(state) => {
+					let mut failures = vec![];
+					for report_equivocations in state {
+						if let Err(failure) =
+							report_equivocations.next(source_client, reporter).await
+						{
+							failures.push(failure);
+						}
+					}
+
+					if !failures.is_empty() {
+						return Err(Self::ReportEquivocations(failures));
+					}
+
+					Ok(())
+				},
+			}
+		}
+		.boxed()
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::mock::*;
+	use std::collections::HashMap;
+
+	impl PartialEq for ReadContext<TestEquivocationDetectionPipeline> {
+		fn eq(&self, other: &Self) -> bool {
+			self.target_block_num == other.target_block_num &&
+				self.synced_headers == other.synced_headers
+		}
+	}
+
+	impl PartialEq for FindEquivocations<TestEquivocationDetectionPipeline> {
+		fn eq(&self, other: &Self) -> bool {
+			self.target_block_num == other.target_block_num &&
+				self.synced_headers == other.synced_headers &&
+				self.context == other.context
+		}
+	}
+
+	impl PartialEq for ReportEquivocations<TestEquivocationDetectionPipeline> {
+		fn eq(&self, other: &Self) -> bool {
+			self.source_block_hash == other.source_block_hash &&
+				self.equivocations == other.equivocations
+		}
+	}
+
+	impl PartialEq for BlockChecker<TestEquivocationDetectionPipeline> {
+		fn eq(&self, _other: &Self) -> bool {
+			matches!(self, _other)
+		}
+	}
+
+	#[async_std::test]
+	async fn block_checker_works() {
+		let mut source_client = TestSourceClient { ..Default::default() };
+		let mut target_client = TestTargetClient {
+			best_synced_header_hash: HashMap::from([(9, Ok(Some(5)))]),
+			finality_verification_context: HashMap::from([(
+				9,
+				Ok(TestFinalityVerificationContext { check_equivocations: true }),
+			)]),
+			synced_headers_finality_info: HashMap::from([(
+				10,
+				Ok(vec![
+					new_header_finality_info(6, None),
+					new_header_finality_info(7, Some(false)),
+					new_header_finality_info(8, None),
+					new_header_finality_info(9, Some(true)),
+					new_header_finality_info(10, None),
+					new_header_finality_info(11, None),
+					new_header_finality_info(12, None),
+				]),
+			)]),
+			..Default::default()
+		};
+		let mut reporter =
+			EquivocationsReporter::<TestEquivocationDetectionPipeline, TestSourceClient>::new();
+
+		let block_checker = BlockChecker::new(10);
+		assert!(block_checker
+			.run(
+				&mut source_client,
+				&mut target_client,
+				&mut FinalityProofsBuf::new(vec![
+					TestFinalityProof(6, vec!["6-1"]),
+					TestFinalityProof(7, vec![]),
+					TestFinalityProof(8, vec!["8-1"]),
+					TestFinalityProof(9, vec!["9-1"]),
+					TestFinalityProof(10, vec![]),
+					TestFinalityProof(11, vec!["11-1", "11-2"]),
+					TestFinalityProof(12, vec!["12-1"])
+				]),
+				&mut reporter
+			)
+			.await
+			.is_ok());
+		assert_eq!(
+			*source_client.reported_equivocations.lock().unwrap(),
+			HashMap::from([(5, vec!["6-1"]), (9, vec!["11-1", "11-2", "12-1"])])
+		);
+	}
+
+	#[async_std::test]
+	async fn block_checker_works_with_empty_context() {
+		let mut target_client = TestTargetClient {
+			best_synced_header_hash: HashMap::from([(9, Ok(None))]),
+			finality_verification_context: HashMap::from([(
+				9,
+				Ok(TestFinalityVerificationContext { check_equivocations: true }),
+			)]),
+			synced_headers_finality_info: HashMap::from([(
+				10,
+				Ok(vec![new_header_finality_info(6, None)]),
+			)]),
+			..Default::default()
+		};
+		let mut source_client = TestSourceClient { ..Default::default() };
+		let mut reporter =
+			EquivocationsReporter::<TestEquivocationDetectionPipeline, TestSourceClient>::new();
+
+		let block_checker = BlockChecker::new(10);
+		assert!(block_checker
+			.run(
+				&mut source_client,
+				&mut target_client,
+				&mut FinalityProofsBuf::new(vec![TestFinalityProof(6, vec!["6-1"])]),
+				&mut reporter
+			)
+			.await
+			.is_ok());
+		assert_eq!(*source_client.reported_equivocations.lock().unwrap(), HashMap::default());
+	}
+
+	#[async_std::test]
+	async fn read_synced_headers_handles_errors() {
+		let mut target_client = TestTargetClient {
+			synced_headers_finality_info: HashMap::from([
+				(10, Err(TestClientError::NonConnection)),
+				(11, Err(TestClientError::Connection)),
+			]),
+			..Default::default()
+		};
+		let mut source_client = TestSourceClient { ..Default::default() };
+		let mut reporter =
+			EquivocationsReporter::<TestEquivocationDetectionPipeline, TestSourceClient>::new();
+
+		// NonConnection error
+		let block_checker = BlockChecker::new(10);
+		assert_eq!(
+			block_checker
+				.run(
+					&mut source_client,
+					&mut target_client,
+					&mut FinalityProofsBuf::new(vec![]),
+					&mut reporter
+				)
+				.await,
+			Err(BlockChecker::ReadSyncedHeaders(ReadSyncedHeaders { target_block_num: 10 }))
+		);
+		assert_eq!(target_client.num_reconnects, 0);
+
+		// Connection error
+		let block_checker = BlockChecker::new(11);
+		assert_eq!(
+			block_checker
+				.run(
+					&mut source_client,
+					&mut target_client,
+					&mut FinalityProofsBuf::new(vec![]),
+					&mut reporter
+				)
+				.await,
+			Err(BlockChecker::ReadSyncedHeaders(ReadSyncedHeaders { target_block_num: 11 }))
+		);
+		assert_eq!(target_client.num_reconnects, 1);
+	}
+
+	#[async_std::test]
+	async fn read_context_handles_errors() {
+		let mut target_client = TestTargetClient {
+			synced_headers_finality_info: HashMap::from([(10, Ok(vec![])), (11, Ok(vec![]))]),
+			best_synced_header_hash: HashMap::from([
+				(9, Err(TestClientError::NonConnection)),
+				(10, Err(TestClientError::Connection)),
+			]),
+			..Default::default()
+		};
+		let mut source_client = TestSourceClient { ..Default::default() };
+		let mut reporter =
+			EquivocationsReporter::<TestEquivocationDetectionPipeline, TestSourceClient>::new();
+
+		// NonConnection error
+		let block_checker = BlockChecker::new(10);
+		assert_eq!(
+			block_checker
+				.run(
+					&mut source_client,
+					&mut target_client,
+					&mut FinalityProofsBuf::new(vec![]),
+					&mut reporter
+				)
+				.await,
+			Err(BlockChecker::ReadContext(ReadContext {
+				target_block_num: 10,
+				synced_headers: vec![]
+			}))
+		);
+		assert_eq!(target_client.num_reconnects, 0);
+
+		// Connection error
+		let block_checker = BlockChecker::new(11);
+		assert_eq!(
+			block_checker
+				.run(
+					&mut source_client,
+					&mut target_client,
+					&mut FinalityProofsBuf::new(vec![]),
+					&mut reporter
+				)
+				.await,
+			Err(BlockChecker::ReadContext(ReadContext {
+				target_block_num: 11,
+				synced_headers: vec![]
+			}))
+		);
+		assert_eq!(target_client.num_reconnects, 1);
+	}
+}

pezbridges/relays/equivocation/src/equivocation_loop.rs

@@ -0,0 +1,310 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{
+	handle_client_error, reporter::EquivocationsReporter, EquivocationDetectionPipeline,
+	SourceClient, TargetClient,
+};
+
+use crate::block_checker::BlockChecker;
+use pez_finality_relay::{FinalityProofsBuf, FinalityProofsStream};
+use futures::{select_biased, FutureExt};
+use num_traits::Saturating;
+use relay_utils::{metrics::MetricsParams, FailedClient};
+use std::{future::Future, time::Duration};
+
+/// Equivocations detection loop state.
+struct EquivocationDetectionLoop<
+	P: EquivocationDetectionPipeline,
+	SC: SourceClient<P>,
+	TC: TargetClient<P>,
+> {
+	source_client: SC,
+	target_client: TC,
+
+	from_block_num: Option<P::TargetNumber>,
+	until_block_num: Option<P::TargetNumber>,
+
+	reporter: EquivocationsReporter<'static, P, SC>,
+
+	finality_proofs_stream: FinalityProofsStream<P, SC>,
+	finality_proofs_buf: FinalityProofsBuf<P>,
+}
+
+impl<P: EquivocationDetectionPipeline, SC: SourceClient<P>, TC: TargetClient<P>>
+	EquivocationDetectionLoop<P, SC, TC>
+{
+	async fn ensure_finality_proofs_stream(&mut self) {
+		match self.finality_proofs_stream.ensure_stream(&self.source_client).await {
+			Ok(_) => {},
+			Err(e) => {
+				tracing::error!(
+					target: "bridge",
+					error=?e,
+					source=%P::SOURCE_NAME,
+					"Could not connect to `FinalityProofsStream`"
+				);
+
+				// Reconnect to the source client if needed
+				handle_client_error(&mut self.source_client, e).await;
+			},
+		}
+	}
+
+	async fn best_finalized_target_block_number(&mut self) -> Option<P::TargetNumber> {
+		match self.target_client.best_finalized_header_number().await {
+			Ok(block_num) => Some(block_num),
+			Err(e) => {
+				tracing::error!(
+					target: "bridge",
+					error=?e,
+					target=%P::TARGET_NAME,
+					"Could not read best finalized header number"
+				);
+
+				// Reconnect target client and move on
+				handle_client_error(&mut self.target_client, e).await;
+
+				None
+			},
+		}
+	}
+
+	async fn do_run(&mut self, tick: Duration, exit_signal: impl Future<Output = ()>) {
+		let exit_signal = exit_signal.fuse();
+		futures::pin_mut!(exit_signal);
+
+		loop {
+			// Make sure that we are connected to the source finality proofs stream.
+			self.ensure_finality_proofs_stream().await;
+			// Check the status of the pending equivocation reports
+			self.reporter.process_pending_reports().await;
+
+			// Update blocks range.
+			if let Some(block_number) = self.best_finalized_target_block_number().await {
+				self.from_block_num.get_or_insert(block_number);
+				self.until_block_num = Some(block_number);
+			}
+			let (from, until) = match (self.from_block_num, self.until_block_num) {
+				(Some(from), Some(until)) => (from, until),
+				_ => continue,
+			};
+
+			// Check the available blocks
+			let mut current_block_number = from;
+			while current_block_number <= until {
+				self.finality_proofs_buf.fill(&mut self.finality_proofs_stream);
+				let block_checker = BlockChecker::new(current_block_number);
+				let _ = block_checker
+					.run(
+						&mut self.source_client,
+						&mut self.target_client,
+						&mut self.finality_proofs_buf,
+						&mut self.reporter,
+					)
+					.await;
+				current_block_number = current_block_number.saturating_add(1.into());
+			}
+			self.from_block_num = Some(current_block_number);
+
+			select_biased! {
+				_ = exit_signal => return,
+				_ = async_std::task::sleep(tick).fuse() => {},
+			}
+		}
+	}
+
+	pub async fn run(
+		source_client: SC,
+		target_client: TC,
+		tick: Duration,
+		exit_signal: impl Future<Output = ()>,
+	) -> Result<(), FailedClient> {
+		let mut equivocation_detection_loop = Self {
+			source_client,
+			target_client,
+			from_block_num: None,
+			until_block_num: None,
+			reporter: EquivocationsReporter::<P, SC>::new(),
+			finality_proofs_stream: FinalityProofsStream::new(),
+			finality_proofs_buf: FinalityProofsBuf::new(vec![]),
+		};
+
+		equivocation_detection_loop.do_run(tick, exit_signal).await;
+		Ok(())
+	}
+}
+
+/// Spawn the equivocations detection loop.
+pub async fn run<P: EquivocationDetectionPipeline>(
+	source_client: impl SourceClient<P>,
+	target_client: impl TargetClient<P>,
+	tick: Duration,
+	metrics_params: MetricsParams,
+	exit_signal: impl Future<Output = ()> + 'static + Send,
+) -> Result<(), relay_utils::Error> {
+	let exit_signal = exit_signal.shared();
+	relay_utils::relay_loop(source_client, target_client)
+		.with_metrics(metrics_params)
+		.expose()
+		.await?
+		.run(
+			format!("{}_to_{}_EquivocationDetection", P::SOURCE_NAME, P::TARGET_NAME),
+			move |source_client, target_client, _metrics| {
+				EquivocationDetectionLoop::run(
+					source_client,
+					target_client,
+					tick,
+					exit_signal.clone(),
+				)
+			},
+		)
+		.await
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::mock::*;
+	use futures::{channel::mpsc::UnboundedSender, StreamExt};
+	use std::{
+		collections::{HashMap, VecDeque},
+		sync::{Arc, Mutex},
+	};
+
+	fn best_finalized_header_number(
+		best_finalized_headers: &Mutex<VecDeque<Result<TestTargetNumber, TestClientError>>>,
+		exit_sender: &UnboundedSender<()>,
+	) -> Result<TestTargetNumber, TestClientError> {
+		let mut best_finalized_headers = best_finalized_headers.lock().unwrap();
+		let result = best_finalized_headers.pop_front().unwrap();
+		if best_finalized_headers.is_empty() {
+			exit_sender.unbounded_send(()).unwrap();
+		}
+		result
+	}
+
+	#[async_std::test]
+	async fn multiple_blocks_are_checked_correctly() {
+		let best_finalized_headers = Arc::new(Mutex::new(VecDeque::from([Ok(10), Ok(12), Ok(13)])));
+		let (exit_sender, exit_receiver) = futures::channel::mpsc::unbounded();
+
+		let source_client = TestSourceClient {
+			finality_proofs: Arc::new(Mutex::new(vec![
+				TestFinalityProof(2, vec!["2-1"]),
+				TestFinalityProof(3, vec!["3-1", "3-2"]),
+				TestFinalityProof(4, vec!["4-1"]),
+				TestFinalityProof(5, vec!["5-1"]),
+				TestFinalityProof(6, vec!["6-1", "6-2"]),
+				TestFinalityProof(7, vec!["7-1", "7-2"]),
+			])),
+			..Default::default()
+		};
+		let reported_equivocations = source_client.reported_equivocations.clone();
+		let target_client = TestTargetClient {
+			best_finalized_header_number: Arc::new(move || {
+				best_finalized_header_number(&best_finalized_headers, &exit_sender)
+			}),
+			best_synced_header_hash: HashMap::from([
+				(9, Ok(Some(1))),
+				(10, Ok(Some(3))),
+				(11, Ok(Some(5))),
+				(12, Ok(Some(6))),
+			]),
+			finality_verification_context: HashMap::from([
+				(9, Ok(TestFinalityVerificationContext { check_equivocations: true })),
+				(10, Ok(TestFinalityVerificationContext { check_equivocations: true })),
+				(11, Ok(TestFinalityVerificationContext { check_equivocations: false })),
+				(12, Ok(TestFinalityVerificationContext { check_equivocations: true })),
+			]),
+			synced_headers_finality_info: HashMap::from([
+				(
+					10,
+					Ok(vec![new_header_finality_info(2, None), new_header_finality_info(3, None)]),
+				),
+				(
+					11,
+					Ok(vec![
+						new_header_finality_info(4, None),
+						new_header_finality_info(5, Some(false)),
+					]),
+				),
+				(12, Ok(vec![new_header_finality_info(6, None)])),
+				(13, Ok(vec![new_header_finality_info(7, None)])),
+			]),
+			..Default::default()
+		};
+
+		assert!(run::<TestEquivocationDetectionPipeline>(
+			source_client,
+			target_client,
+			Duration::from_secs(0),
+			MetricsParams { address: None, registry: Default::default() },
+			exit_receiver.into_future().map(|(_, _)| ()),
+		)
+		.await
+		.is_ok());
+		assert_eq!(
+			*reported_equivocations.lock().unwrap(),
+			HashMap::from([
+				(1, vec!["2-1", "3-1", "3-2"]),
+				(3, vec!["4-1", "5-1"]),
+				(6, vec!["7-1", "7-2"])
+			])
+		);
+	}
+
+	#[async_std::test]
+	async fn blocks_following_error_are_checked_correctly() {
+		let best_finalized_headers = Mutex::new(VecDeque::from([Ok(10), Ok(11)]));
+		let (exit_sender, exit_receiver) = futures::channel::mpsc::unbounded();
+
+		let source_client = TestSourceClient {
+			finality_proofs: Arc::new(Mutex::new(vec![
+				TestFinalityProof(2, vec!["2-1"]),
+				TestFinalityProof(3, vec!["3-1"]),
+			])),
+			..Default::default()
+		};
+		let reported_equivocations = source_client.reported_equivocations.clone();
+		let target_client = TestTargetClient {
+			best_finalized_header_number: Arc::new(move || {
+				best_finalized_header_number(&best_finalized_headers, &exit_sender)
+			}),
+			best_synced_header_hash: HashMap::from([(9, Ok(Some(1))), (10, Ok(Some(2)))]),
+			finality_verification_context: HashMap::from([
+				(9, Ok(TestFinalityVerificationContext { check_equivocations: true })),
+				(10, Ok(TestFinalityVerificationContext { check_equivocations: true })),
+			]),
+			synced_headers_finality_info: HashMap::from([
+				(10, Err(TestClientError::NonConnection)),
+				(11, Ok(vec![new_header_finality_info(3, None)])),
+			]),
+			..Default::default()
+		};
+
+		assert!(run::<TestEquivocationDetectionPipeline>(
+			source_client,
+			target_client,
+			Duration::from_secs(0),
+			MetricsParams { address: None, registry: Default::default() },
+			exit_receiver.into_future().map(|(_, _)| ()),
+		)
+		.await
+		.is_ok());
+		assert_eq!(*reported_equivocations.lock().unwrap(), HashMap::from([(2, vec!["3-1"]),]));
+	}
+}

pezbridges/relays/equivocation/src/lib.rs

@@ -0,0 +1,137 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+mod block_checker;
+mod equivocation_loop;
+mod mock;
+mod reporter;
+
+use async_trait::async_trait;
+use bp_header_pez_chain::{FinalityProof, FindEquivocations};
+use pez_finality_relay::{FinalityPipeline, SourceClientBase};
+use relay_utils::{relay_loop::Client as RelayClient, MaybeConnectionError, TransactionTracker};
+use std::{fmt::Debug, time::Duration};
+
+pub use equivocation_loop::run;
+
+#[cfg(not(test))]
+const RECONNECT_DELAY: Duration = relay_utils::relay_loop::RECONNECT_DELAY;
+#[cfg(test)]
+const RECONNECT_DELAY: Duration = mock::TEST_RECONNECT_DELAY;
+
+pub trait EquivocationDetectionPipeline: FinalityPipeline {
+	/// Block number of the target chain.
+	type TargetNumber: relay_utils::BlockNumberBase;
+	/// The context needed for validating finality proofs.
+	type FinalityVerificationContext: Debug + Send;
+	/// The type of the equivocation proof.
+	type EquivocationProof: Clone + Debug + Send + Sync;
+	/// The equivocations finder.
+	type EquivocationsFinder: FindEquivocations<
+		Self::FinalityProof,
+		Self::FinalityVerificationContext,
+		Self::EquivocationProof,
+	>;
+}
+
+type HeaderFinalityInfo<P> = bp_header_pez_chain::HeaderFinalityInfo<
+	<P as FinalityPipeline>::FinalityProof,
+	<P as EquivocationDetectionPipeline>::FinalityVerificationContext,
+>;
+
+/// Source client used in equivocation detection loop.
+#[async_trait]
+pub trait SourceClient<P: EquivocationDetectionPipeline>: SourceClientBase<P> {
+	/// Transaction tracker to track submitted transactions.
+	type TransactionTracker: TransactionTracker;
+
+	/// Report equivocation.
+	async fn report_equivocation(
+		&self,
+		at: P::Hash,
+		equivocation: P::EquivocationProof,
+	) -> Result<Self::TransactionTracker, Self::Error>;
+}
+
+/// Target client used in equivocation detection loop.
+#[async_trait]
+pub trait TargetClient<P: EquivocationDetectionPipeline>: RelayClient {
+	/// Get the best finalized header number.
+	async fn best_finalized_header_number(&self) -> Result<P::TargetNumber, Self::Error>;
+
+	/// Get the hash of the best source header known by the target at the provided block number.
+	async fn best_synced_header_hash(
+		&self,
+		at: P::TargetNumber,
+	) -> Result<Option<P::Hash>, Self::Error>;
+
+	/// Get the data stored by the target at the specified block for validating source finality
+	/// proofs.
+	async fn finality_verification_context(
+		&self,
+		at: P::TargetNumber,
+	) -> Result<P::FinalityVerificationContext, Self::Error>;
+
+	/// Get the finality info associated to the source headers synced with the target chain at the
+	/// specified block.
+	async fn synced_headers_finality_info(
+		&self,
+		at: P::TargetNumber,
+	) -> Result<Vec<HeaderFinalityInfo<P>>, Self::Error>;
+}
+
+/// The context needed for finding equivocations inside finality proofs and reporting them.
+#[derive(Debug, PartialEq)]
+struct EquivocationReportingContext<P: EquivocationDetectionPipeline> {
+	pub synced_header_hash: P::Hash,
+	pub synced_verification_context: P::FinalityVerificationContext,
+}
+
+impl<P: EquivocationDetectionPipeline> EquivocationReportingContext<P> {
+	/// Try to get the `EquivocationReportingContext` used by the target chain
+	/// at the provided block.
+	pub async fn try_read_from_target<TC: TargetClient<P>>(
+		target_client: &TC,
+		at: P::TargetNumber,
+	) -> Result<Option<Self>, TC::Error> {
+		let maybe_best_synced_header_hash = target_client.best_synced_header_hash(at).await?;
+		Ok(match maybe_best_synced_header_hash {
+			Some(best_synced_header_hash) => Some(EquivocationReportingContext {
+				synced_header_hash: best_synced_header_hash,
+				synced_verification_context: target_client
+					.finality_verification_context(at)
+					.await?,
+			}),
+			None => None,
+		})
+	}
+
+	/// Update with the new context introduced by the `HeaderFinalityInfo<P>` if any.
+	pub fn update(&mut self, info: HeaderFinalityInfo<P>) {
+		if let Some(new_verification_context) = info.new_verification_context {
+			self.synced_header_hash = info.finality_proof.target_header_hash();
+			self.synced_verification_context = new_verification_context;
+		}
+	}
+}
+
+async fn handle_client_error<C: RelayClient>(client: &mut C, e: C::Error) {
+	if e.is_connection_error() {
+		client.reconnect_until_success(RECONNECT_DELAY).await;
+	} else {
+		async_std::task::sleep(RECONNECT_DELAY).await;
+	}
+}

pezbridges/relays/equivocation/src/mock.rs

@@ -0,0 +1,285 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+#![cfg(test)]
+
+use crate::{EquivocationDetectionPipeline, HeaderFinalityInfo, SourceClient, TargetClient};
+use async_trait::async_trait;
+use bp_header_pez_chain::{FinalityProof, FindEquivocations};
+use pez_finality_relay::{FinalityPipeline, SourceClientBase};
+use futures::{Stream, StreamExt};
+use relay_utils::{
+	relay_loop::Client as RelayClient, HeaderId, MaybeConnectionError, TrackedTransactionStatus,
+	TransactionTracker,
+};
+use std::{
+	collections::HashMap,
+	pin::Pin,
+	sync::{Arc, Mutex},
+	time::Duration,
+};
+
+pub type TestSourceHashAndNumber = u64;
+pub type TestTargetNumber = u64;
+pub type TestEquivocationProof = &'static str;
+
+pub const TEST_RECONNECT_DELAY: Duration = Duration::from_secs(0);
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TestFinalityProof(pub TestSourceHashAndNumber, pub Vec<TestEquivocationProof>);
+
+impl FinalityProof<TestSourceHashAndNumber, TestSourceHashAndNumber> for TestFinalityProof {
+	fn target_header_hash(&self) -> TestSourceHashAndNumber {
+		self.0
+	}
+
+	fn target_header_number(&self) -> TestSourceHashAndNumber {
+		self.0
+	}
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct TestEquivocationDetectionPipeline;
+
+impl FinalityPipeline for TestEquivocationDetectionPipeline {
+	const SOURCE_NAME: &'static str = "TestSource";
+	const TARGET_NAME: &'static str = "TestTarget";
+
+	type Hash = TestSourceHashAndNumber;
+	type Number = TestSourceHashAndNumber;
+	type FinalityProof = TestFinalityProof;
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct TestFinalityVerificationContext {
+	pub check_equivocations: bool,
+}
+
+pub struct TestEquivocationsFinder;
+
+impl FindEquivocations<TestFinalityProof, TestFinalityVerificationContext, TestEquivocationProof>
+	for TestEquivocationsFinder
+{
+	type Error = ();
+
+	fn find_equivocations(
+		verification_context: &TestFinalityVerificationContext,
+		synced_proof: &TestFinalityProof,
+		source_proofs: &[TestFinalityProof],
+	) -> Result<Vec<TestEquivocationProof>, Self::Error> {
+		if verification_context.check_equivocations {
+			// Get the equivocations from the source proofs, in order to make sure
+			// that they are correctly provided.
+			if let Some(proof) = source_proofs.iter().find(|proof| proof.0 == synced_proof.0) {
+				return Ok(proof.1.clone());
+			}
+		}
+
+		Ok(vec![])
+	}
+}
+
+impl EquivocationDetectionPipeline for TestEquivocationDetectionPipeline {
+	type TargetNumber = TestTargetNumber;
+	type FinalityVerificationContext = TestFinalityVerificationContext;
+	type EquivocationProof = TestEquivocationProof;
+	type EquivocationsFinder = TestEquivocationsFinder;
+}
+
+#[derive(Debug, Clone)]
+pub enum TestClientError {
+	Connection,
+	NonConnection,
+}
+
+impl MaybeConnectionError for TestClientError {
+	fn is_connection_error(&self) -> bool {
+		match self {
+			TestClientError::Connection => true,
+			TestClientError::NonConnection => false,
+		}
+	}
+}
+
+#[derive(Clone)]
+pub struct TestSourceClient {
+	pub num_reconnects: u32,
+	pub finality_proofs: Arc<Mutex<Vec<TestFinalityProof>>>,
+	pub reported_equivocations:
+		Arc<Mutex<HashMap<TestSourceHashAndNumber, Vec<TestEquivocationProof>>>>,
+}
+
+impl Default for TestSourceClient {
+	fn default() -> Self {
+		Self {
+			num_reconnects: 0,
+			finality_proofs: Arc::new(Mutex::new(vec![])),
+			reported_equivocations: Arc::new(Mutex::new(Default::default())),
+		}
+	}
+}
+
+#[async_trait]
+impl RelayClient for TestSourceClient {
+	type Error = TestClientError;
+
+	async fn reconnect(&mut self) -> Result<(), Self::Error> {
+		self.num_reconnects += 1;
+
+		Ok(())
+	}
+}
+
+#[async_trait]
+impl SourceClientBase<TestEquivocationDetectionPipeline> for TestSourceClient {
+	type FinalityProofsStream = Pin<Box<dyn Stream<Item = TestFinalityProof> + 'static + Send>>;
+
+	async fn finality_proofs(&self) -> Result<Self::FinalityProofsStream, Self::Error> {
+		let finality_proofs = std::mem::take(&mut *self.finality_proofs.lock().unwrap());
+		Ok(futures::stream::iter(finality_proofs).boxed())
+	}
+}
+
+#[derive(Clone, Debug)]
+pub struct TestTransactionTracker(
+	pub TrackedTransactionStatus<HeaderId<TestSourceHashAndNumber, TestSourceHashAndNumber>>,
+);
+
+impl Default for TestTransactionTracker {
+	fn default() -> TestTransactionTracker {
+		TestTransactionTracker(TrackedTransactionStatus::Finalized(Default::default()))
+	}
+}
+
+#[async_trait]
+impl TransactionTracker for TestTransactionTracker {
+	type HeaderId = HeaderId<TestSourceHashAndNumber, TestSourceHashAndNumber>;
+
+	async fn wait(
+		self,
+	) -> TrackedTransactionStatus<HeaderId<TestSourceHashAndNumber, TestSourceHashAndNumber>> {
+		self.0
+	}
+}
+
+#[async_trait]
+impl SourceClient<TestEquivocationDetectionPipeline> for TestSourceClient {
+	type TransactionTracker = TestTransactionTracker;
+
+	async fn report_equivocation(
+		&self,
+		at: TestSourceHashAndNumber,
+		equivocation: TestEquivocationProof,
+	) -> Result<Self::TransactionTracker, Self::Error> {
+		self.reported_equivocations
+			.lock()
+			.unwrap()
+			.entry(at)
+			.or_default()
+			.push(equivocation);
+
+		Ok(TestTransactionTracker::default())
+	}
+}
+
+#[derive(Clone)]
+pub struct TestTargetClient {
+	pub num_reconnects: u32,
+	pub best_finalized_header_number:
+		Arc<dyn Fn() -> Result<TestTargetNumber, TestClientError> + Send + Sync>,
+	pub best_synced_header_hash:
+		HashMap<TestTargetNumber, Result<Option<TestSourceHashAndNumber>, TestClientError>>,
+	pub finality_verification_context:
+		HashMap<TestTargetNumber, Result<TestFinalityVerificationContext, TestClientError>>,
+	pub synced_headers_finality_info: HashMap<
+		TestTargetNumber,
+		Result<Vec<HeaderFinalityInfo<TestEquivocationDetectionPipeline>>, TestClientError>,
+	>,
+}
+
+impl Default for TestTargetClient {
+	fn default() -> Self {
+		Self {
+			num_reconnects: 0,
+			best_finalized_header_number: Arc::new(|| Ok(0)),
+			best_synced_header_hash: Default::default(),
+			finality_verification_context: Default::default(),
+			synced_headers_finality_info: Default::default(),
+		}
+	}
+}
+
+#[async_trait]
+impl RelayClient for TestTargetClient {
+	type Error = TestClientError;
+
+	async fn reconnect(&mut self) -> Result<(), Self::Error> {
+		self.num_reconnects += 1;
+
+		Ok(())
+	}
+}
+
+#[async_trait]
+impl TargetClient<TestEquivocationDetectionPipeline> for TestTargetClient {
+	async fn best_finalized_header_number(&self) -> Result<TestTargetNumber, Self::Error> {
+		(self.best_finalized_header_number)()
+	}
+
+	async fn best_synced_header_hash(
+		&self,
+		at: TestTargetNumber,
+	) -> Result<Option<TestSourceHashAndNumber>, Self::Error> {
+		self.best_synced_header_hash
+			.get(&at)
+			.unwrap_or(&Err(TestClientError::NonConnection))
+			.clone()
+	}
+
+	async fn finality_verification_context(
+		&self,
+		at: TestTargetNumber,
+	) -> Result<TestFinalityVerificationContext, Self::Error> {
+		self.finality_verification_context
+			.get(&at)
+			.unwrap_or(&Err(TestClientError::NonConnection))
+			.clone()
+	}
+
+	async fn synced_headers_finality_info(
+		&self,
+		at: TestTargetNumber,
+	) -> Result<Vec<HeaderFinalityInfo<TestEquivocationDetectionPipeline>>, Self::Error> {
+		self.synced_headers_finality_info
+			.get(&at)
+			.unwrap_or(&Err(TestClientError::NonConnection))
+			.clone()
+	}
+}
+
+pub fn new_header_finality_info(
+	source_hdr: TestSourceHashAndNumber,
+	check_following_equivocations: Option<bool>,
+) -> HeaderFinalityInfo<TestEquivocationDetectionPipeline> {
+	HeaderFinalityInfo::<TestEquivocationDetectionPipeline> {
+		finality_proof: TestFinalityProof(source_hdr, vec![]),
+		new_verification_context: check_following_equivocations.map(
+			|check_following_equivocations| TestFinalityVerificationContext {
+				check_equivocations: check_following_equivocations,
+			},
+		),
+	}
+}

pezbridges/relays/equivocation/src/reporter.rs

@@ -0,0 +1,129 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Helper struct used for submitting finality reports and tracking their status.
+
+use crate::{EquivocationDetectionPipeline, SourceClient};
+
+use futures::FutureExt;
+use relay_utils::{TrackedTransactionFuture, TrackedTransactionStatus, TransactionTracker};
+use std::{
+	future::poll_fn,
+	task::{Context, Poll},
+};
+
+pub struct EquivocationsReporter<'a, P: EquivocationDetectionPipeline, SC: SourceClient<P>> {
+	pending_reports: Vec<TrackedTransactionFuture<'a, SC::TransactionTracker>>,
+}
+
+impl<'a, P: EquivocationDetectionPipeline, SC: SourceClient<P>> EquivocationsReporter<'a, P, SC> {
+	pub fn new() -> Self {
+		Self { pending_reports: vec![] }
+	}
+
+	/// Submit a `report_equivocation()` transaction to the source chain.
+	///
+	/// We store the transaction tracker for future monitoring.
+	pub async fn submit_report(
+		&mut self,
+		source_client: &SC,
+		at: P::Hash,
+		equivocation: P::EquivocationProof,
+	) -> Result<(), SC::Error> {
+		let pending_report = source_client.report_equivocation(at, equivocation).await?;
+		self.pending_reports.push(pending_report.wait());
+
+		Ok(())
+	}
+
+	fn do_process_pending_reports(&mut self, cx: &mut Context<'_>) -> Poll<()> {
+		self.pending_reports.retain_mut(|pending_report| {
+			match pending_report.poll_unpin(cx) {
+				Poll::Ready(tx_status) => {
+					match tx_status {
+						TrackedTransactionStatus::Lost => {
+							tracing::error!(target: "bridge", "Equivocation report tx was lost");
+						},
+						TrackedTransactionStatus::Finalized(id) => {
+							tracing::error!(target: "bridge", ?id, "Equivocation report tx was finalized in source block");
+						},
+					}
+
+					// The future was processed. Drop it.
+					false
+				},
+				Poll::Pending => {
+					// The future is still pending. Retain it.
+					true
+				},
+			}
+		});
+
+		Poll::Ready(())
+	}
+
+	/// Iterate through all the pending `report_equivocation()` transactions
+	/// and log the ones that finished.
+	pub async fn process_pending_reports(&mut self) {
+		poll_fn(|cx| self.do_process_pending_reports(cx)).await
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::mock::*;
+	use relay_utils::HeaderId;
+	use std::sync::Mutex;
+
+	#[async_std::test]
+	async fn process_pending_reports_works() {
+		let polled_reports = Mutex::new(vec![]);
+		let finished_reports = Mutex::new(vec![]);
+
+		let mut reporter =
+			EquivocationsReporter::<TestEquivocationDetectionPipeline, TestSourceClient> {
+				pending_reports: vec![
+					Box::pin(async {
+						polled_reports.lock().unwrap().push(1);
+						finished_reports.lock().unwrap().push(1);
+						TrackedTransactionStatus::Finalized(HeaderId(1, 1))
+					}),
+					Box::pin(async {
+						polled_reports.lock().unwrap().push(2);
+						finished_reports.lock().unwrap().push(2);
+						TrackedTransactionStatus::Finalized(HeaderId(2, 2))
+					}),
+					Box::pin(async {
+						polled_reports.lock().unwrap().push(3);
+						std::future::pending::<()>().await;
+						finished_reports.lock().unwrap().push(3);
+						TrackedTransactionStatus::Finalized(HeaderId(3, 3))
+					}),
+					Box::pin(async {
+						polled_reports.lock().unwrap().push(4);
+						finished_reports.lock().unwrap().push(4);
+						TrackedTransactionStatus::Finalized(HeaderId(4, 4))
+					}),
+				],
+			};
+
+		reporter.process_pending_reports().await;
+		assert_eq!(*polled_reports.lock().unwrap(), vec![1, 2, 3, 4]);
+		assert_eq!(*finished_reports.lock().unwrap(), vec![1, 2, 4]);
+		assert_eq!(reporter.pending_reports.len(), 1);
+	}
+}

pezbridges/relays/finality/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "pez-finality-relay"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+repository.workspace = true
+description = "Finality proofs relay"
+publish = false
+documentation = "https://docs.rs/pez-finality-relay"
+homepage = { workspace = true }
+
+[lints]
+workspace = true
+
+[dependencies]
+async-std = { workspace = true }
+async-trait = { workspace = true }
+backoff = { workspace = true }
+bp-header-pez-chain = { workspace = true, default-features = true }
+futures = { workspace = true }
+num-traits = { workspace = true, default-features = true }
+relay-utils = { workspace = true }
+tracing = { workspace = true }
+
+[dev-dependencies]
+parking_lot = { workspace = true, default-features = true }
+
+[features]
+runtime-benchmarks = [
+  "bp-header-pez-chain/runtime-benchmarks",
+  "relay-utils/runtime-benchmarks",
+]

pezbridges/relays/finality/README.md

@@ -0,0 +1,62 @@
+# GRANDPA Finality Relay
+
+The finality relay is able to work with different finality engines. In the modern Bizinikiwi world they are GRANDPA
+and BEEFY. Let's talk about GRANDPA here, because BEEFY relay and bridge BEEFY pezpallet are in development.
+
+In general, the relay works as follows: it connects to the source and target chain. The source chain must have the
+[GRANDPA gadget](https://github.com/pezkuwichain/finality-grandpa) running (so it can't be a teyrchain). The target
+chain must have the [bridge GRANDPA pezpallet](../../modules/grandpa/) deployed at its runtime. The relay subscribes
+to the GRANDPA finality notifications at the source chain and when the new justification is received, it is submitted
+to the pezpallet at the target chain.
+
+Apart from that, the relay is watching for every source header that is missing at target. If it finds the missing
+mandatory header (header that is changing the current GRANDPA validators set), it submits the justification for
+this header. The case when the source node can't return the mandatory justification is considered a fatal error,
+because the pezpallet can't proceed without it.
+
+More: [GRANDPA Finality Relay Sequence Diagram](../../docs/grandpa-pez-finality-relay.html).
+
+## How to Use the Finality Relay
+
+The most important trait is the [`FinalitySyncPipeline`](./src/lib.rs), which defines the basic primitives of the
+source chain (like block hash and number) and the type of finality proof (GRANDPA justification or MMR proof). Once
+that is defined, there are two other traits - [`SourceClient`](./src/finality_loop.rs) and
+[`TargetClient`](./src/finality_loop.rs).
+
+The `SourceClient` represents the Bizinikiwi node client that connects to the source chain. The client needs to
+be able to return the best finalized header number, finalized header and its finality proof and the stream of
+finality proofs.
+
+The `TargetClient` implementation must be able to craft finality delivery transaction and submit it to the target
+node. The transaction is then tracked by the relay until it is mined and finalized.
+
+The main entrypoint for the crate is the [`run` function](./src/finality_loop.rs), which takes source and target
+clients and [`FinalitySyncParams`](./src/finality_loop.rs) parameters. The most important parameter is the
+`only_mandatory_headers` - it is set to `true`, the relay will only submit mandatory headers. Since transactions
+with mandatory headers are fee-free, the cost of running such relay is zero (in terms of fees). If a similar,
+`only_free_headers` parameter, is set to `true`, then free headers (if configured in the runtime) are also
+relayed.
+
+## Finality Relay Metrics
+
+Finality relay provides several metrics. Metrics names depend on names of source and target chains. The list below
+shows metrics names for pezkuwichain (source chain) to BridgeHubzagros (target chain) finality relay. For other
+chains, simply change chain names. So the metrics are:
+
+- `pezkuwichain_to_BridgeHubzagros_Sync_best_source_block_number` - returns best finalized source chain (pezkuwichain) block
+  number, known to the relay.
+  If relay is running in [on-demand mode](../bin-bizinikiwi/src/cli/relay_headers_and_messages/), the
+  number may not match (it may be far behind) the actual best finalized number;
+
+- `pezkuwichain_to_BridgeHubzagros_Sync_best_source_at_target_block_number` - returns best finalized source chain (pezkuwichain)
+  block number that is known to the bridge GRANDPA pezpallet at the target chain.
+
+- `pezkuwichain_to_BridgeHubzagros_Sync_is_source_and_source_at_target_using_different_forks` - if this metrics is set
+  to `1`, then  the best source chain header known to the target chain doesn't match the same-number-header
+  at the source chain. It means that the GRANDPA validators set has crafted the duplicate justification
+  and it has been submitted to the target chain.
+  Normally (if majority of validators are honest and if you're running finality relay without large breaks)
+  this shall not happen and the metric will have `0` value.
+
+If relay operates properly, you should see that the `pezkuwichain_to_BridgeHubzagros_Sync_best_source_at_target_block_number`
+tries to reach the `pezkuwichain_to_BridgeHubzagros_Sync_best_source_block_number`. And the latter one always increases.

pezbridges/relays/finality/src/base.rs

@@ -0,0 +1,47 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use async_trait::async_trait;
+use bp_header_pez_chain::FinalityProof;
+use futures::Stream;
+use relay_utils::relay_loop::Client as RelayClient;
+use std::fmt::Debug;
+
+/// Base finality pipeline.
+pub trait FinalityPipeline: 'static + Clone + Debug + Send + Sync {
+	/// Name of the finality proofs source.
+	const SOURCE_NAME: &'static str;
+	/// Name of the finality proofs target.
+	const TARGET_NAME: &'static str;
+
+	/// Synced headers are identified by this hash.
+	type Hash: Eq + Clone + Copy + Send + Sync + Debug;
+	/// Synced headers are identified by this number.
+	type Number: relay_utils::BlockNumberBase;
+	/// Finality proof type.
+	type FinalityProof: FinalityProof<Self::Hash, Self::Number>;
+}
+
+/// Source client used in finality related loops.
+#[async_trait]
+pub trait SourceClientBase<P: FinalityPipeline>: RelayClient {
+	/// Stream of new finality proofs. The stream is allowed to miss proofs for some
+	/// headers, even if those headers are mandatory.
+	type FinalityProofsStream: Stream<Item = P::FinalityProof> + Send + Unpin;
+
+	/// Subscribe to new finality proofs.
+	async fn finality_proofs(&self) -> Result<Self::FinalityProofsStream, Self::Error>;
+}

pezbridges/relays/finality/src/finality_loop.rs

@@ -0,0 +1,797 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The loop basically reads all missing headers and their finality proofs from the source client.
+//! The proof for the best possible header is then submitted to the target node. The only exception
+//! is the mandatory headers, which we always submit to the target node. For such headers, we
+//! assume that the persistent proof either exists, or will eventually become available.
+
+use crate::{sync_loop_metrics::SyncLoopMetrics, Error, FinalitySyncPipeline, SourceHeader};
+
+use crate::{
+	base::SourceClientBase,
+	finality_proofs::{FinalityProofsBuf, FinalityProofsStream},
+	headers::{JustifiedHeader, JustifiedHeaderSelector},
+};
+use async_trait::async_trait;
+use backoff::{backoff::Backoff, ExponentialBackoff};
+use futures::{future::Fuse, select, Future, FutureExt};
+use num_traits::{Saturating, Zero};
+use relay_utils::{
+	metrics::MetricsParams, relay_loop::Client as RelayClient, retry_backoff, FailedClient,
+	HeaderId, MaybeConnectionError, TrackedTransactionStatus, TransactionTracker,
+};
+use std::{
+	fmt::Debug,
+	time::{Duration, Instant},
+};
+
+/// Type of headers that we relay.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum HeadersToRelay {
+	/// Relay all headers.
+	All,
+	/// Relay only mandatory headers.
+	Mandatory,
+	/// Relay only free (including mandatory) headers.
+	Free,
+}
+
+/// Finality proof synchronization loop parameters.
+#[derive(Debug, Clone)]
+pub struct FinalitySyncParams {
+	/// Interval at which we check updates on both clients. Normally should be larger than
+	/// `min(source_block_time, target_block_time)`.
+	///
+	/// This parameter may be used to limit transactions rate. Increase the value && you'll get
+	/// infrequent updates => sparse headers => potential slow down of bridge applications, but
+	/// pezpallet storage won't be super large. Decrease the value to near `source_block_time` and
+	/// you'll get transaction for (almost) every block of the source chain => all source headers
+	/// will be known to the target chain => bridge applications will run faster, but pezpallet
+	/// storage may explode (but if pruning is there, then it's fine).
+	pub tick: Duration,
+	/// Number of finality proofs to keep in internal buffer between loop iterations.
+	///
+	/// While in "major syncing" state, we still read finality proofs from the stream. They're
+	/// stored in the internal buffer between loop iterations. When we're close to the tip of the
+	/// chain, we may meet finality delays if headers are not finalized frequently. So instead of
+	/// waiting for next finality proof to appear in the stream, we may use existing proof from
+	/// that buffer.
+	pub recent_finality_proofs_limit: usize,
+	/// Timeout before we treat our transactions as lost and restart the whole sync process.
+	pub stall_timeout: Duration,
+	/// If true, only mandatory headers are relayed.
+	pub headers_to_relay: HeadersToRelay,
+}
+
+/// Source client used in finality synchronization loop.
+#[async_trait]
+pub trait SourceClient<P: FinalitySyncPipeline>: SourceClientBase<P> {
+	/// Get best finalized block number.
+	async fn best_finalized_block_number(&self) -> Result<P::Number, Self::Error>;
+
+	/// Get canonical header and its finality proof by number.
+	async fn header_and_finality_proof(
+		&self,
+		number: P::Number,
+	) -> Result<(P::Header, Option<P::FinalityProof>), Self::Error>;
+}
+
+/// Target client used in finality synchronization loop.
+#[async_trait]
+pub trait TargetClient<P: FinalitySyncPipeline>: RelayClient {
+	/// Transaction tracker to track submitted transactions.
+	type TransactionTracker: TransactionTracker;
+
+	/// Get best finalized source block number.
+	async fn best_finalized_source_block_id(
+		&self,
+	) -> Result<HeaderId<P::Hash, P::Number>, Self::Error>;
+
+	/// Get free source headers submission interval, if it is configured in the
+	/// target runtime.
+	async fn free_source_headers_interval(&self) -> Result<Option<P::Number>, Self::Error>;
+
+	/// Submit header finality proof.
+	async fn submit_finality_proof(
+		&self,
+		header: P::Header,
+		proof: P::FinalityProof,
+		is_free_execution_expected: bool,
+	) -> Result<Self::TransactionTracker, Self::Error>;
+}
+
+/// Return prefix that will be used by default to expose Prometheus metrics of the finality proofs
+/// sync loop.
+pub fn metrics_prefix<P: FinalitySyncPipeline>() -> String {
+	format!("{}_to_{}_Sync", P::SOURCE_NAME, P::TARGET_NAME)
+}
+
+/// Finality sync information.
+pub struct SyncInfo<P: FinalitySyncPipeline> {
+	/// Best finalized header at the source client.
+	pub best_number_at_source: P::Number,
+	/// Best source header, known to the target client.
+	pub best_number_at_target: P::Number,
+	/// Whether the target client follows the same fork as the source client do.
+	pub is_using_same_fork: bool,
+}
+
+impl<P: FinalitySyncPipeline> SyncInfo<P> {
+	/// Checks if both clients are on the same fork.
+	async fn is_on_same_fork<SC: SourceClient<P>>(
+		source_client: &SC,
+		id_at_target: &HeaderId<P::Hash, P::Number>,
+	) -> Result<bool, SC::Error> {
+		let header_at_source = source_client.header_and_finality_proof(id_at_target.0).await?.0;
+		let header_hash_at_source = header_at_source.hash();
+		Ok(if id_at_target.1 == header_hash_at_source {
+			true
+		} else {
+			tracing::error!(
+				target: "bridge",
+				source=%P::SOURCE_NAME,
+				target=%P::TARGET_NAME,
+				height=?id_at_target.0,
+				at_source=?header_hash_at_source,
+				at_target=?id_at_target.1,
+				"Source node and pezpallet at target node have different headers at the same height"
+			);
+
+			false
+		})
+	}
+
+	async fn new<SC: SourceClient<P>, TC: TargetClient<P>>(
+		source_client: &SC,
+		target_client: &TC,
+	) -> Result<Self, Error<P, SC::Error, TC::Error>> {
+		let best_number_at_source =
+			source_client.best_finalized_block_number().await.map_err(Error::Source)?;
+		let best_id_at_target =
+			target_client.best_finalized_source_block_id().await.map_err(Error::Target)?;
+		let best_number_at_target = best_id_at_target.0;
+
+		let is_using_same_fork = Self::is_on_same_fork(source_client, &best_id_at_target)
+			.await
+			.map_err(Error::Source)?;
+
+		Ok(Self { best_number_at_source, best_number_at_target, is_using_same_fork })
+	}
+
+	fn update_metrics(&self, metrics_sync: &Option<SyncLoopMetrics>) {
+		if let Some(metrics_sync) = metrics_sync {
+			metrics_sync.update_best_block_at_source(self.best_number_at_source);
+			metrics_sync.update_best_block_at_target(self.best_number_at_target);
+			metrics_sync.update_using_same_fork(self.is_using_same_fork);
+		}
+	}
+
+	pub fn num_headers(&self) -> P::Number {
+		self.best_number_at_source.saturating_sub(self.best_number_at_target)
+	}
+}
+
+/// Information about transaction that we have submitted.
+#[derive(Debug, Clone)]
+pub struct Transaction<Tracker, Number> {
+	/// Submitted transaction tracker.
+	tracker: Tracker,
+	/// The number of the header we have submitted.
+	header_number: Number,
+}
+
+impl<Tracker: TransactionTracker, Number: Debug + PartialOrd> Transaction<Tracker, Number> {
+	pub async fn submit<
+		P: FinalitySyncPipeline<Number = Number>,
+		TC: TargetClient<P, TransactionTracker = Tracker>,
+	>(
+		target_client: &TC,
+		header: P::Header,
+		justification: P::FinalityProof,
+		is_free_execution_expected: bool,
+	) -> Result<Self, TC::Error> {
+		let header_number = header.number();
+		tracing::debug!(
+			target: "bridge",
+			source=%P::SOURCE_NAME,
+			target=%P::TARGET_NAME,
+			header=?header_number,
+			"Going to submit finality proof of header"
+		);
+
+		let tracker = target_client
+			.submit_finality_proof(header, justification, is_free_execution_expected)
+			.await?;
+		Ok(Transaction { tracker, header_number })
+	}
+
+	async fn track<
+		P: FinalitySyncPipeline<Number = Number>,
+		SC: SourceClient<P>,
+		TC: TargetClient<P>,
+	>(
+		self,
+		target_client: TC,
+	) -> Result<(), Error<P, SC::Error, TC::Error>> {
+		match self.tracker.wait().await {
+			TrackedTransactionStatus::Finalized(_) => {
+				// The transaction has been finalized, but it may have been finalized in the
+				// "failed" state. So let's check if the block number was actually updated.
+				target_client
+					.best_finalized_source_block_id()
+					.await
+					.map_err(Error::Target)
+					.and_then(|best_id_at_target| {
+						if self.header_number > best_id_at_target.0 {
+							return Err(Error::ProofSubmissionTxFailed {
+								submitted_number: self.header_number,
+								best_number_at_target: best_id_at_target.0,
+							});
+						}
+						Ok(())
+					})
+			},
+			TrackedTransactionStatus::Lost => Err(Error::ProofSubmissionTxLost),
+		}
+	}
+}
+
+/// Finality synchronization loop state.
+struct FinalityLoop<P: FinalitySyncPipeline, SC: SourceClient<P>, TC: TargetClient<P>> {
+	source_client: SC,
+	target_client: TC,
+
+	sync_params: FinalitySyncParams,
+	metrics_sync: Option<SyncLoopMetrics>,
+
+	progress: (Instant, Option<P::Number>),
+	retry_backoff: ExponentialBackoff,
+	finality_proofs_stream: FinalityProofsStream<P, SC>,
+	finality_proofs_buf: FinalityProofsBuf<P>,
+	best_submitted_number: Option<P::Number>,
+}
+
+impl<P: FinalitySyncPipeline, SC: SourceClient<P>, TC: TargetClient<P>> FinalityLoop<P, SC, TC> {
+	pub fn new(
+		source_client: SC,
+		target_client: TC,
+		sync_params: FinalitySyncParams,
+		metrics_sync: Option<SyncLoopMetrics>,
+	) -> Self {
+		Self {
+			source_client,
+			target_client,
+			sync_params,
+			metrics_sync,
+			progress: (Instant::now(), None),
+			retry_backoff: retry_backoff(),
+			finality_proofs_stream: FinalityProofsStream::new(),
+			finality_proofs_buf: FinalityProofsBuf::new(vec![]),
+			best_submitted_number: None,
+		}
+	}
+
+	fn update_progress(&mut self, info: &SyncInfo<P>) {
+		let (prev_time, prev_best_number_at_target) = self.progress;
+		let now = Instant::now();
+
+		let needs_update = now - prev_time > Duration::from_secs(10) ||
+			prev_best_number_at_target
+				.map(|prev_best_number_at_target| {
+					info.best_number_at_target.saturating_sub(prev_best_number_at_target) >
+						10.into()
+				})
+				.unwrap_or(true);
+
+		if !needs_update {
+			return;
+		}
+
+		tracing::info!(
+			target: "bridge",
+			best_number_at_target=?info.best_number_at_target,
+			best_number_at_source=?info.best_number_at_source,
+			"Synced headers"
+		);
+
+		self.progress = (now, Some(info.best_number_at_target))
+	}
+
+	pub async fn select_header_to_submit(
+		&mut self,
+		info: &SyncInfo<P>,
+		free_headers_interval: Option<P::Number>,
+	) -> Result<Option<JustifiedHeader<P>>, Error<P, SC::Error, TC::Error>> {
+		// to see that the loop is progressing
+		tracing::trace!(
+			target: "bridge",
+			best_number_at_target=%info.best_number_at_target,
+			best_number_at_source=%info.best_number_at_source,
+			"Considering range of headers"
+		);
+
+		// read missing headers
+		let selector = JustifiedHeaderSelector::new::<SC, TC>(
+			&self.source_client,
+			info,
+			self.sync_params.headers_to_relay,
+			free_headers_interval,
+		)
+		.await?;
+		// if we see that the header schedules GRANDPA change, we need to submit it
+		if self.sync_params.headers_to_relay == HeadersToRelay::Mandatory {
+			return Ok(selector.select_mandatory());
+		}
+
+		// all headers that are missing from the target client are non-mandatory
+		// => even if we have already selected some header and its persistent finality proof,
+		// we may try to select better header by reading non-persistent proofs from the stream
+		self.finality_proofs_buf.fill(&mut self.finality_proofs_stream);
+		let maybe_justified_header = selector.select(
+			info,
+			self.sync_params.headers_to_relay,
+			free_headers_interval,
+			&self.finality_proofs_buf,
+		);
+
+		// remove obsolete 'recent' finality proofs + keep its size under certain limit
+		let oldest_finality_proof_to_keep = maybe_justified_header
+			.as_ref()
+			.map(|justified_header| justified_header.number())
+			.unwrap_or(info.best_number_at_target);
+		self.finality_proofs_buf.prune(
+			oldest_finality_proof_to_keep,
+			Some(self.sync_params.recent_finality_proofs_limit),
+		);
+
+		Ok(maybe_justified_header)
+	}
+
+	pub async fn run_iteration(
+		&mut self,
+		free_headers_interval: Option<P::Number>,
+	) -> Result<
+		Option<Transaction<TC::TransactionTracker, P::Number>>,
+		Error<P, SC::Error, TC::Error>,
+	> {
+		// read best source headers ids from source and target nodes
+		let info = SyncInfo::new(&self.source_client, &self.target_client).await?;
+		info.update_metrics(&self.metrics_sync);
+		self.update_progress(&info);
+
+		// if we have already submitted header, then we just need to wait for it
+		// if we're waiting too much, then we believe our transaction has been lost and restart sync
+		if Some(info.best_number_at_target) < self.best_submitted_number {
+			return Ok(None);
+		}
+
+		// submit new header if we have something new
+		match self.select_header_to_submit(&info, free_headers_interval).await? {
+			Some(header) => {
+				let transaction = Transaction::submit(
+					&self.target_client,
+					header.header,
+					header.proof,
+					self.sync_params.headers_to_relay == HeadersToRelay::Free,
+				)
+				.await
+				.map_err(Error::Target)?;
+				self.best_submitted_number = Some(transaction.header_number);
+				Ok(Some(transaction))
+			},
+			None => Ok(None),
+		}
+	}
+
+	async fn ensure_finality_proofs_stream(&mut self) -> Result<(), FailedClient> {
+		if let Err(e) = self.finality_proofs_stream.ensure_stream(&self.source_client).await {
+			if e.is_connection_error() {
+				return Err(FailedClient::Source);
+			}
+		}
+
+		Ok(())
+	}
+
+	/// Run finality relay loop until connection to one of nodes is lost.
+	async fn run_until_connection_lost(
+		&mut self,
+		exit_signal: impl Future<Output = ()>,
+	) -> Result<(), FailedClient> {
+		self.ensure_finality_proofs_stream().await?;
+		let proof_submission_tx_tracker = Fuse::terminated();
+		let exit_signal = exit_signal.fuse();
+		futures::pin_mut!(exit_signal, proof_submission_tx_tracker);
+
+		let free_headers_interval = free_headers_interval(&self.target_client).await?;
+
+		loop {
+			// run loop iteration
+			let next_tick = match self.run_iteration(free_headers_interval).await {
+				Ok(Some(tx)) => {
+					proof_submission_tx_tracker
+						.set(tx.track::<P, SC, _>(self.target_client.clone()).fuse());
+					self.retry_backoff.reset();
+					self.sync_params.tick
+				},
+				Ok(None) => {
+					self.retry_backoff.reset();
+					self.sync_params.tick
+				},
+				Err(error) => {
+					tracing::error!(target: "bridge", ?error, "Finality sync loop iteration has failed");
+					error.fail_if_connection_error()?;
+					self.retry_backoff
+						.next_backoff()
+						.unwrap_or(relay_utils::relay_loop::RECONNECT_DELAY)
+				},
+			};
+			self.ensure_finality_proofs_stream().await?;
+
+			// wait till exit signal, or new source block
+			select! {
+				proof_submission_result = proof_submission_tx_tracker => {
+					if let Err(e) = proof_submission_result {
+						tracing::error!(
+							target: "bridge",
+							error=?e,
+							target=%P::TARGET_NAME,
+							"Finality sync proof submission tx has failed."
+						);
+						self.best_submitted_number = None;
+						e.fail_if_connection_error()?;
+					}
+				},
+				_ = async_std::task::sleep(next_tick).fuse() => {},
+				_ = exit_signal => return Ok(()),
+			}
+		}
+	}
+
+	pub async fn run(
+		source_client: SC,
+		target_client: TC,
+		sync_params: FinalitySyncParams,
+		metrics_sync: Option<SyncLoopMetrics>,
+		exit_signal: impl Future<Output = ()>,
+	) -> Result<(), FailedClient> {
+		let mut finality_loop = Self::new(source_client, target_client, sync_params, metrics_sync);
+		finality_loop.run_until_connection_lost(exit_signal).await
+	}
+}
+
+async fn free_headers_interval<P: FinalitySyncPipeline>(
+	target_client: &impl TargetClient<P>,
+) -> Result<Option<P::Number>, FailedClient> {
+	match target_client.free_source_headers_interval().await {
+		Ok(Some(free_headers_interval)) if !free_headers_interval.is_zero() => {
+			tracing::trace!(
+				target: "bridge",
+				source=%P::SOURCE_NAME,
+				target=%P::TARGET_NAME,
+				?free_headers_interval,
+				"Free headers interval for headers"
+			);
+			Ok(Some(free_headers_interval))
+		},
+		Ok(Some(_free_headers_interval)) => {
+			tracing::trace!(
+				target: "bridge",
+				source=%P::SOURCE_NAME,
+				target=%P::TARGET_NAME,
+				"Free headers interval for headers is zero. Not submitting any free headers"
+			);
+			Ok(None)
+		},
+		Ok(None) => {
+			tracing::trace!(
+				target: "bridge",
+				source=%P::SOURCE_NAME,
+				target=%P::TARGET_NAME,
+				"Free headers interval for headers is None. Not submitting any free headers"
+			);
+
+			Ok(None)
+		},
+		Err(e) => {
+			tracing::error!(
+				target: "bridge",
+				error=?e,
+				source=%P::SOURCE_NAME,
+				target=%P::TARGET_NAME,
+				"Failed to read free headers interval for headers"
+			);
+			Err(FailedClient::Target)
+		},
+	}
+}
+
+/// Run finality proofs synchronization loop.
+pub async fn run<P: FinalitySyncPipeline>(
+	source_client: impl SourceClient<P>,
+	target_client: impl TargetClient<P>,
+	sync_params: FinalitySyncParams,
+	metrics_params: MetricsParams,
+	exit_signal: impl Future<Output = ()> + 'static + Send,
+) -> Result<(), relay_utils::Error> {
+	let exit_signal = exit_signal.shared();
+	relay_utils::relay_loop(source_client, target_client)
+		.with_metrics(metrics_params)
+		.loop_metric(SyncLoopMetrics::new(
+			Some(&metrics_prefix::<P>()),
+			"source",
+			"source_at_target",
+		)?)?
+		.expose()
+		.await?
+		.run(metrics_prefix::<P>(), move |source_client, target_client, metrics| {
+			FinalityLoop::run(
+				source_client,
+				target_client,
+				sync_params.clone(),
+				metrics,
+				exit_signal.clone(),
+			)
+		})
+		.await
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	use crate::mock::*;
+	use futures::{FutureExt, StreamExt};
+	use parking_lot::Mutex;
+	use relay_utils::{FailedClient, HeaderId, TrackedTransactionStatus};
+	use std::{collections::HashMap, sync::Arc};
+
+	fn prepare_test_clients(
+		exit_sender: futures::channel::mpsc::UnboundedSender<()>,
+		state_function: impl Fn(&mut ClientsData) -> bool + Send + Sync + 'static,
+		source_headers: HashMap<TestNumber, (TestSourceHeader, Option<TestFinalityProof>)>,
+	) -> (TestSourceClient, TestTargetClient) {
+		let internal_state_function: Arc<dyn Fn(&mut ClientsData) + Send + Sync> =
+			Arc::new(move |data| {
+				if state_function(data) {
+					exit_sender.unbounded_send(()).unwrap();
+				}
+			});
+		let clients_data = Arc::new(Mutex::new(ClientsData {
+			source_best_block_number: 10,
+			source_headers,
+			source_proofs: vec![TestFinalityProof(12), TestFinalityProof(14)],
+
+			target_best_block_id: HeaderId(5, 5),
+			target_headers: vec![],
+			target_transaction_tracker: TestTransactionTracker(
+				TrackedTransactionStatus::Finalized(Default::default()),
+			),
+		}));
+		(
+			TestSourceClient {
+				on_method_call: internal_state_function.clone(),
+				data: clients_data.clone(),
+			},
+			TestTargetClient { on_method_call: internal_state_function, data: clients_data },
+		)
+	}
+
+	fn test_sync_params() -> FinalitySyncParams {
+		FinalitySyncParams {
+			tick: Duration::from_secs(0),
+			recent_finality_proofs_limit: 1024,
+			stall_timeout: Duration::from_secs(1),
+			headers_to_relay: HeadersToRelay::All,
+		}
+	}
+
+	fn run_sync_loop(
+		state_function: impl Fn(&mut ClientsData) -> bool + Send + Sync + 'static,
+	) -> (ClientsData, Result<(), FailedClient>) {
+		let (exit_sender, exit_receiver) = futures::channel::mpsc::unbounded();
+		let (source_client, target_client) = prepare_test_clients(
+			exit_sender,
+			state_function,
+			vec![
+				(5, (TestSourceHeader(false, 5, 5), None)),
+				(6, (TestSourceHeader(false, 6, 6), None)),
+				(7, (TestSourceHeader(false, 7, 7), Some(TestFinalityProof(7)))),
+				(8, (TestSourceHeader(true, 8, 8), Some(TestFinalityProof(8)))),
+				(9, (TestSourceHeader(false, 9, 9), Some(TestFinalityProof(9)))),
+				(10, (TestSourceHeader(false, 10, 10), None)),
+			]
+			.into_iter()
+			.collect(),
+		);
+		let sync_params = test_sync_params();
+
+		let clients_data = source_client.data.clone();
+		let result = async_std::task::block_on(FinalityLoop::run(
+			source_client,
+			target_client,
+			sync_params,
+			None,
+			exit_receiver.into_future().map(|(_, _)| ()),
+		));
+
+		let clients_data = clients_data.lock().clone();
+		(clients_data, result)
+	}
+
+	#[test]
+	fn finality_sync_loop_works() {
+		let (client_data, result) = run_sync_loop(|data| {
+			// header#7 has persistent finality proof, but it isn't mandatory => it isn't submitted,
+			// because header#8 has persistent finality proof && it is mandatory => it is submitted
+			// header#9 has persistent finality proof, but it isn't mandatory => it is submitted,
+			// because   there are no more persistent finality proofs
+			//
+			// once this ^^^ is done, we generate more blocks && read proof for blocks 12 and 14
+			// from the stream
+			if data.target_best_block_id.0 == 9 {
+				data.source_best_block_number = 14;
+				data.source_headers.insert(11, (TestSourceHeader(false, 11, 11), None));
+				data.source_headers
+					.insert(12, (TestSourceHeader(false, 12, 12), Some(TestFinalityProof(12))));
+				data.source_headers.insert(13, (TestSourceHeader(false, 13, 13), None));
+				data.source_headers
+					.insert(14, (TestSourceHeader(false, 14, 14), Some(TestFinalityProof(14))));
+			}
+			// once this ^^^ is done, we generate more blocks && read persistent proof for block 16
+			if data.target_best_block_id.0 == 14 {
+				data.source_best_block_number = 17;
+				data.source_headers.insert(15, (TestSourceHeader(false, 15, 15), None));
+				data.source_headers
+					.insert(16, (TestSourceHeader(false, 16, 16), Some(TestFinalityProof(16))));
+				data.source_headers.insert(17, (TestSourceHeader(false, 17, 17), None));
+			}
+
+			data.target_best_block_id.0 == 16
+		});
+
+		assert_eq!(result, Ok(()));
+		assert_eq!(
+			client_data.target_headers,
+			vec![
+				// before adding 11..14: finality proof for mandatory header#8
+				(TestSourceHeader(true, 8, 8), TestFinalityProof(8)),
+				// before adding 11..14: persistent finality proof for non-mandatory header#9
+				(TestSourceHeader(false, 9, 9), TestFinalityProof(9)),
+				// after adding 11..14: ephemeral finality proof for non-mandatory header#14
+				(TestSourceHeader(false, 14, 14), TestFinalityProof(14)),
+				// after adding 15..17: persistent finality proof for non-mandatory header#16
+				(TestSourceHeader(false, 16, 16), TestFinalityProof(16)),
+			],
+		);
+	}
+
+	fn run_headers_to_relay_mode_test(
+		headers_to_relay: HeadersToRelay,
+		has_mandatory_headers: bool,
+	) -> Option<JustifiedHeader<TestFinalitySyncPipeline>> {
+		let (exit_sender, _) = futures::channel::mpsc::unbounded();
+		let (source_client, target_client) = prepare_test_clients(
+			exit_sender,
+			|_| false,
+			vec![
+				(6, (TestSourceHeader(false, 6, 6), Some(TestFinalityProof(6)))),
+				(7, (TestSourceHeader(false, 7, 7), Some(TestFinalityProof(7)))),
+				(8, (TestSourceHeader(has_mandatory_headers, 8, 8), Some(TestFinalityProof(8)))),
+				(9, (TestSourceHeader(false, 9, 9), Some(TestFinalityProof(9)))),
+				(10, (TestSourceHeader(false, 10, 10), Some(TestFinalityProof(10)))),
+			]
+			.into_iter()
+			.collect(),
+		);
+		async_std::task::block_on(async {
+			let mut finality_loop = FinalityLoop::new(
+				source_client,
+				target_client,
+				FinalitySyncParams {
+					tick: Duration::from_secs(0),
+					recent_finality_proofs_limit: 0,
+					stall_timeout: Duration::from_secs(0),
+					headers_to_relay,
+				},
+				None,
+			);
+			let info = SyncInfo {
+				best_number_at_source: 10,
+				best_number_at_target: 5,
+				is_using_same_fork: true,
+			};
+			finality_loop.select_header_to_submit(&info, Some(3)).await.unwrap()
+		})
+	}
+
+	#[test]
+	fn select_header_to_submit_may_select_non_mandatory_header() {
+		assert_eq!(run_headers_to_relay_mode_test(HeadersToRelay::Mandatory, false), None);
+		assert_eq!(
+			run_headers_to_relay_mode_test(HeadersToRelay::Free, false),
+			Some(JustifiedHeader {
+				header: TestSourceHeader(false, 10, 10),
+				proof: TestFinalityProof(10)
+			}),
+		);
+		assert_eq!(
+			run_headers_to_relay_mode_test(HeadersToRelay::All, false),
+			Some(JustifiedHeader {
+				header: TestSourceHeader(false, 10, 10),
+				proof: TestFinalityProof(10)
+			}),
+		);
+	}
+
+	#[test]
+	fn select_header_to_submit_may_select_mandatory_header() {
+		assert_eq!(
+			run_headers_to_relay_mode_test(HeadersToRelay::Mandatory, true),
+			Some(JustifiedHeader {
+				header: TestSourceHeader(true, 8, 8),
+				proof: TestFinalityProof(8)
+			}),
+		);
+		assert_eq!(
+			run_headers_to_relay_mode_test(HeadersToRelay::Free, true),
+			Some(JustifiedHeader {
+				header: TestSourceHeader(true, 8, 8),
+				proof: TestFinalityProof(8)
+			}),
+		);
+		assert_eq!(
+			run_headers_to_relay_mode_test(HeadersToRelay::All, true),
+			Some(JustifiedHeader {
+				header: TestSourceHeader(true, 8, 8),
+				proof: TestFinalityProof(8)
+			}),
+		);
+	}
+
+	#[test]
+	fn different_forks_at_source_and_at_target_are_detected() {
+		let (exit_sender, _exit_receiver) = futures::channel::mpsc::unbounded();
+		let (source_client, target_client) = prepare_test_clients(
+			exit_sender,
+			|_| false,
+			vec![
+				(5, (TestSourceHeader(false, 5, 42), None)),
+				(6, (TestSourceHeader(false, 6, 6), None)),
+				(7, (TestSourceHeader(false, 7, 7), None)),
+				(8, (TestSourceHeader(false, 8, 8), None)),
+				(9, (TestSourceHeader(false, 9, 9), None)),
+				(10, (TestSourceHeader(false, 10, 10), None)),
+			]
+			.into_iter()
+			.collect(),
+		);
+
+		let metrics_sync = SyncLoopMetrics::new(None, "source", "target").unwrap();
+		async_std::task::block_on(async {
+			let mut finality_loop = FinalityLoop::new(
+				source_client,
+				target_client,
+				test_sync_params(),
+				Some(metrics_sync.clone()),
+			);
+			finality_loop.run_iteration(None).await.unwrap()
+		});
+
+		assert!(!metrics_sync.is_using_same_fork());
+	}
+}

pezbridges/relays/finality/src/finality_proofs.rs

@@ -0,0 +1,221 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{base::SourceClientBase, FinalityPipeline};
+
+use bp_header_pez_chain::FinalityProof;
+use futures::{FutureExt, Stream, StreamExt};
+use std::pin::Pin;
+
+/// Source finality proofs stream that may be restarted.
+#[derive(Default)]
+pub struct FinalityProofsStream<P: FinalityPipeline, SC: SourceClientBase<P>> {
+	/// The underlying stream.
+	stream: Option<Pin<Box<SC::FinalityProofsStream>>>,
+}
+
+impl<P: FinalityPipeline, SC: SourceClientBase<P>> FinalityProofsStream<P, SC> {
+	pub fn new() -> Self {
+		Self { stream: None }
+	}
+
+	pub fn from_stream(stream: SC::FinalityProofsStream) -> Self {
+		Self { stream: Some(Box::pin(stream)) }
+	}
+
+	fn next(&mut self) -> Option<<SC::FinalityProofsStream as Stream>::Item> {
+		let stream = match &mut self.stream {
+			Some(stream) => stream,
+			None => return None,
+		};
+
+		match stream.next().now_or_never() {
+			Some(Some(finality_proof)) => Some(finality_proof),
+			Some(None) => {
+				self.stream = None;
+				None
+			},
+			None => None,
+		}
+	}
+
+	pub async fn ensure_stream(&mut self, source_client: &SC) -> Result<(), SC::Error> {
+		if self.stream.is_none() {
+			tracing::warn!(target: "bridge", source=%P::SOURCE_NAME, "Finality proofs stream is being started / restarted");
+
+			let stream = source_client.finality_proofs().await.map_err(|error| {
+				tracing::error!(
+					target: "bridge",
+					?error,
+					source=%P::SOURCE_NAME,
+					"Failed to subscribe to justifications"
+				);
+
+				error
+			})?;
+			self.stream = Some(Box::pin(stream));
+		}
+
+		Ok(())
+	}
+}
+
+/// Source finality proofs buffer.
+pub struct FinalityProofsBuf<P: FinalityPipeline> {
+	/// Proofs buffer. Ordered by target header number.
+	buf: Vec<P::FinalityProof>,
+}
+
+impl<P: FinalityPipeline> FinalityProofsBuf<P> {
+	pub fn new(buf: Vec<P::FinalityProof>) -> Self {
+		Self { buf }
+	}
+
+	pub fn buf(&self) -> &Vec<P::FinalityProof> {
+		&self.buf
+	}
+
+	pub fn fill<SC: SourceClientBase<P>>(&mut self, stream: &mut FinalityProofsStream<P, SC>) {
+		let mut proofs_count = 0;
+		let mut first_header_number = None;
+		let mut last_header_number = None;
+		while let Some(finality_proof) = stream.next() {
+			let target_header_number = finality_proof.target_header_number();
+			first_header_number.get_or_insert(target_header_number);
+			last_header_number = Some(target_header_number);
+			proofs_count += 1;
+
+			self.buf.push(finality_proof);
+		}
+
+		if proofs_count != 0 {
+			tracing::trace!(
+				target: "bridge",
+				source=%P::SOURCE_NAME,
+				%proofs_count,
+				?first_header_number,
+				?last_header_number,
+				"Read finality proofs from finality stream for headers in range",
+			);
+		}
+	}
+
+	/// Prune all finality proofs that target header numbers older than `first_to_keep`.
+	pub fn prune(&mut self, first_to_keep: P::Number, maybe_buf_limit: Option<usize>) {
+		let first_to_keep_idx = self
+			.buf
+			.binary_search_by_key(&first_to_keep, |hdr| hdr.target_header_number())
+			.map(|idx| idx + 1)
+			.unwrap_or_else(|idx| idx);
+		let buf_limit_idx = match maybe_buf_limit {
+			Some(buf_limit) => self.buf.len().saturating_sub(buf_limit),
+			None => 0,
+		};
+
+		self.buf = self.buf.split_off(std::cmp::max(first_to_keep_idx, buf_limit_idx));
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::mock::*;
+
+	#[test]
+	fn finality_proofs_buf_fill_works() {
+		// when stream is currently empty, nothing is changed
+		let mut finality_proofs_buf =
+			FinalityProofsBuf::<TestFinalitySyncPipeline> { buf: vec![TestFinalityProof(1)] };
+		let mut stream =
+			FinalityProofsStream::<TestFinalitySyncPipeline, TestSourceClient>::from_stream(
+				Box::pin(futures::stream::pending()),
+			);
+		finality_proofs_buf.fill(&mut stream);
+		assert_eq!(finality_proofs_buf.buf, vec![TestFinalityProof(1)]);
+		assert!(stream.stream.is_some());
+
+		// when stream has entry with target, it is added to the recent proofs container
+		let mut stream =
+			FinalityProofsStream::<TestFinalitySyncPipeline, TestSourceClient>::from_stream(
+				Box::pin(
+					futures::stream::iter(vec![TestFinalityProof(4)])
+						.chain(futures::stream::pending()),
+				),
+			);
+		finality_proofs_buf.fill(&mut stream);
+		assert_eq!(finality_proofs_buf.buf, vec![TestFinalityProof(1), TestFinalityProof(4)]);
+		assert!(stream.stream.is_some());
+
+		// when stream has ended, we'll need to restart it
+		let mut stream =
+			FinalityProofsStream::<TestFinalitySyncPipeline, TestSourceClient>::from_stream(
+				Box::pin(futures::stream::empty()),
+			);
+		finality_proofs_buf.fill(&mut stream);
+		assert_eq!(finality_proofs_buf.buf, vec![TestFinalityProof(1), TestFinalityProof(4)]);
+		assert!(stream.stream.is_none());
+	}
+
+	#[test]
+	fn finality_proofs_buf_prune_works() {
+		let original_finality_proofs_buf: Vec<
+			<TestFinalitySyncPipeline as FinalityPipeline>::FinalityProof,
+		> = vec![
+			TestFinalityProof(10),
+			TestFinalityProof(13),
+			TestFinalityProof(15),
+			TestFinalityProof(17),
+			TestFinalityProof(19),
+		]
+		.into_iter()
+		.collect();
+
+		// when there's proof for justified header in the vec
+		let mut finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline> {
+			buf: original_finality_proofs_buf.clone(),
+		};
+		finality_proofs_buf.prune(10, None);
+		assert_eq!(&original_finality_proofs_buf[1..], finality_proofs_buf.buf,);
+
+		// when there are no proof for justified header in the vec
+		let mut finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline> {
+			buf: original_finality_proofs_buf.clone(),
+		};
+		finality_proofs_buf.prune(11, None);
+		assert_eq!(&original_finality_proofs_buf[1..], finality_proofs_buf.buf,);
+
+		// when there are too many entries after initial prune && they also need to be pruned
+		let mut finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline> {
+			buf: original_finality_proofs_buf.clone(),
+		};
+		finality_proofs_buf.prune(10, Some(2));
+		assert_eq!(&original_finality_proofs_buf[3..], finality_proofs_buf.buf,);
+
+		// when last entry is pruned
+		let mut finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline> {
+			buf: original_finality_proofs_buf.clone(),
+		};
+		finality_proofs_buf.prune(19, Some(2));
+		assert_eq!(&original_finality_proofs_buf[5..], finality_proofs_buf.buf,);
+
+		// when post-last entry is pruned
+		let mut finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline> {
+			buf: original_finality_proofs_buf.clone(),
+		};
+		finality_proofs_buf.prune(20, Some(2));
+		assert_eq!(&original_finality_proofs_buf[5..], finality_proofs_buf.buf,);
+	}
+}

pezbridges/relays/finality/src/headers.rs

@@ -0,0 +1,361 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{
+	finality_loop::SyncInfo, finality_proofs::FinalityProofsBuf, Error, FinalitySyncPipeline,
+	HeadersToRelay, SourceClient, SourceHeader, TargetClient,
+};
+
+use bp_header_pez_chain::FinalityProof;
+use num_traits::Saturating;
+use std::cmp::Ordering;
+
+/// Unjustified headers container. Ordered by header number.
+pub type UnjustifiedHeaders<H> = Vec<H>;
+
+#[derive(Debug)]
+#[cfg_attr(test, derive(Clone, PartialEq))]
+pub struct JustifiedHeader<P: FinalitySyncPipeline> {
+	pub header: P::Header,
+	pub proof: P::FinalityProof,
+}
+
+impl<P: FinalitySyncPipeline> JustifiedHeader<P> {
+	pub fn number(&self) -> P::Number {
+		self.header.number()
+	}
+}
+
+/// Finality proof that has been selected by the `read_missing_headers` function.
+pub enum JustifiedHeaderSelector<P: FinalitySyncPipeline> {
+	/// Mandatory header and its proof has been selected. We shall submit proof for this header.
+	Mandatory(JustifiedHeader<P>),
+	/// Regular header and its proof has been selected. We may submit this proof, or proof for
+	/// some better header.
+	Regular(UnjustifiedHeaders<P::Header>, JustifiedHeader<P>),
+	/// We haven't found any missing header with persistent proof at the target client.
+	None(UnjustifiedHeaders<P::Header>),
+}
+
+impl<P: FinalitySyncPipeline> JustifiedHeaderSelector<P> {
+	/// Selects last header with persistent justification, missing from the target and matching
+	/// the `headers_to_relay` criteria.
+	pub(crate) async fn new<SC: SourceClient<P>, TC: TargetClient<P>>(
+		source_client: &SC,
+		info: &SyncInfo<P>,
+		headers_to_relay: HeadersToRelay,
+		free_headers_interval: Option<P::Number>,
+	) -> Result<Self, Error<P, SC::Error, TC::Error>> {
+		let mut unjustified_headers = Vec::new();
+		let mut maybe_justified_header = None;
+
+		let mut header_number = info.best_number_at_target + 1.into();
+		while header_number <= info.best_number_at_source {
+			let (header, maybe_proof) = source_client
+				.header_and_finality_proof(header_number)
+				.await
+				.map_err(Error::Source)?;
+
+			match (header.is_mandatory(), maybe_proof) {
+				(true, Some(proof)) => {
+					tracing::trace!(target: "bridge", ?header_number, "Header is mandatory");
+					return Ok(Self::Mandatory(JustifiedHeader { header, proof }));
+				},
+				(true, None) => return Err(Error::MissingMandatoryFinalityProof(header.number())),
+				(false, Some(proof))
+					if need_to_relay::<P>(
+						info,
+						headers_to_relay,
+						free_headers_interval,
+						&header,
+					) =>
+				{
+					tracing::trace!(target: "bridge", ?header_number, "Header has persistent finality proof");
+					unjustified_headers.clear();
+					maybe_justified_header = Some(JustifiedHeader { header, proof });
+				},
+				_ => {
+					unjustified_headers.push(header);
+				},
+			}
+
+			header_number = header_number + 1.into();
+		}
+
+		tracing::trace!(
+			target: "bridge",
+			source=%P::SOURCE_NAME,
+			num_headers=%info.num_headers(),
+			justified_header=?maybe_justified_header.as_ref().map(|justified_header| &justified_header.header),
+			"Read headers. Selected finality proof for header"
+		);
+
+		Ok(match maybe_justified_header {
+			Some(justified_header) => Self::Regular(unjustified_headers, justified_header),
+			None => Self::None(unjustified_headers),
+		})
+	}
+
+	/// Returns selected mandatory header if we have seen one. Otherwise returns `None`.
+	pub fn select_mandatory(self) -> Option<JustifiedHeader<P>> {
+		match self {
+			JustifiedHeaderSelector::Mandatory(header) => Some(header),
+			_ => None,
+		}
+	}
+
+	/// Tries to improve previously selected header using ephemeral
+	/// justifications stream.
+	pub fn select(
+		self,
+		info: &SyncInfo<P>,
+		headers_to_relay: HeadersToRelay,
+		free_headers_interval: Option<P::Number>,
+		buf: &FinalityProofsBuf<P>,
+	) -> Option<JustifiedHeader<P>> {
+		let (unjustified_headers, maybe_justified_header) = match self {
+			JustifiedHeaderSelector::Mandatory(justified_header) => return Some(justified_header),
+			JustifiedHeaderSelector::Regular(unjustified_headers, justified_header) =>
+				(unjustified_headers, Some(justified_header)),
+			JustifiedHeaderSelector::None(unjustified_headers) => (unjustified_headers, None),
+		};
+
+		let mut finality_proofs_iter = buf.buf().iter().rev();
+		let mut maybe_finality_proof = finality_proofs_iter.next();
+
+		let mut unjustified_headers_iter = unjustified_headers.iter().rev();
+		let mut maybe_unjustified_header = unjustified_headers_iter.next();
+
+		while let (Some(finality_proof), Some(unjustified_header)) =
+			(maybe_finality_proof, maybe_unjustified_header)
+		{
+			match finality_proof.target_header_number().cmp(&unjustified_header.number()) {
+				Ordering::Equal
+					if need_to_relay::<P>(
+						info,
+						headers_to_relay,
+						free_headers_interval,
+						&unjustified_header,
+					) =>
+				{
+					tracing::trace!(
+						target: "bridge",
+						source=%P::SOURCE_NAME,
+						justified_header=?maybe_justified_header.as_ref().map(|justified_header| justified_header.number()),
+						target_header_number=?finality_proof.target_header_number(),
+						"Managed to improve selected finality proof."
+					);
+					return Some(JustifiedHeader {
+						header: unjustified_header.clone(),
+						proof: finality_proof.clone(),
+					});
+				},
+				Ordering::Equal => {
+					maybe_finality_proof = finality_proofs_iter.next();
+					maybe_unjustified_header = unjustified_headers_iter.next();
+				},
+				Ordering::Less => maybe_unjustified_header = unjustified_headers_iter.next(),
+				Ordering::Greater => {
+					maybe_finality_proof = finality_proofs_iter.next();
+				},
+			}
+		}
+
+		tracing::trace!(
+			target: "bridge",
+			source=%P::SOURCE_NAME,
+			justified_header=?maybe_justified_header.as_ref().map(|justified_header| justified_header.number()),
+			"Could not improve selected finality proof."
+		);
+		maybe_justified_header
+	}
+}
+
+/// Returns true if we want to relay header `header_number`.
+fn need_to_relay<P: FinalitySyncPipeline>(
+	info: &SyncInfo<P>,
+	headers_to_relay: HeadersToRelay,
+	free_headers_interval: Option<P::Number>,
+	header: &P::Header,
+) -> bool {
+	match headers_to_relay {
+		HeadersToRelay::All => true,
+		HeadersToRelay::Mandatory => header.is_mandatory(),
+		HeadersToRelay::Free =>
+			header.is_mandatory() ||
+				free_headers_interval
+					.map(|free_headers_interval| {
+						header.number().saturating_sub(info.best_number_at_target) >=
+							free_headers_interval
+					})
+					.unwrap_or(false),
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::mock::*;
+
+	#[test]
+	fn select_better_recent_finality_proof_works() {
+		let info = SyncInfo {
+			best_number_at_source: 10,
+			best_number_at_target: 5,
+			is_using_same_fork: true,
+		};
+
+		// if there are no unjustified headers, nothing is changed
+		let finality_proofs_buf =
+			FinalityProofsBuf::<TestFinalitySyncPipeline>::new(vec![TestFinalityProof(5)]);
+		let justified_header =
+			JustifiedHeader { header: TestSourceHeader(false, 2, 2), proof: TestFinalityProof(2) };
+		let selector = JustifiedHeaderSelector::Regular(vec![], justified_header.clone());
+		assert_eq!(
+			selector.select(&info, HeadersToRelay::All, None, &finality_proofs_buf),
+			Some(justified_header)
+		);
+
+		// if there are no buffered finality proofs, nothing is changed
+		let finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline>::new(vec![]);
+		let justified_header =
+			JustifiedHeader { header: TestSourceHeader(false, 2, 2), proof: TestFinalityProof(2) };
+		let selector = JustifiedHeaderSelector::Regular(
+			vec![TestSourceHeader(false, 5, 5)],
+			justified_header.clone(),
+		);
+		assert_eq!(
+			selector.select(&info, HeadersToRelay::All, None, &finality_proofs_buf),
+			Some(justified_header)
+		);
+
+		// if there's no intersection between recent finality proofs and unjustified headers,
+		// nothing is changed
+		let finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline>::new(vec![
+			TestFinalityProof(1),
+			TestFinalityProof(4),
+		]);
+		let justified_header =
+			JustifiedHeader { header: TestSourceHeader(false, 2, 2), proof: TestFinalityProof(2) };
+		let selector = JustifiedHeaderSelector::Regular(
+			vec![TestSourceHeader(false, 9, 9), TestSourceHeader(false, 10, 10)],
+			justified_header.clone(),
+		);
+		assert_eq!(
+			selector.select(&info, HeadersToRelay::All, None, &finality_proofs_buf),
+			Some(justified_header)
+		);
+
+		// if there's intersection between recent finality proofs and unjustified headers, but there
+		// are no proofs in this intersection, nothing is changed
+		let finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline>::new(vec![
+			TestFinalityProof(7),
+			TestFinalityProof(11),
+		]);
+		let justified_header =
+			JustifiedHeader { header: TestSourceHeader(false, 2, 2), proof: TestFinalityProof(2) };
+		let selector = JustifiedHeaderSelector::Regular(
+			vec![
+				TestSourceHeader(false, 8, 8),
+				TestSourceHeader(false, 9, 9),
+				TestSourceHeader(false, 10, 10),
+			],
+			justified_header.clone(),
+		);
+		assert_eq!(
+			selector.select(&info, HeadersToRelay::All, None, &finality_proofs_buf),
+			Some(justified_header)
+		);
+
+		// if there's intersection between recent finality proofs and unjustified headers and
+		// there's a proof in this intersection:
+		// - this better (last from intersection) proof is selected;
+		// - 'obsolete' unjustified headers are pruned.
+		let finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline>::new(vec![
+			TestFinalityProof(7),
+			TestFinalityProof(9),
+		]);
+		let justified_header =
+			JustifiedHeader { header: TestSourceHeader(false, 2, 2), proof: TestFinalityProof(2) };
+		let selector = JustifiedHeaderSelector::Regular(
+			vec![
+				TestSourceHeader(false, 8, 8),
+				TestSourceHeader(false, 9, 9),
+				TestSourceHeader(false, 10, 10),
+			],
+			justified_header,
+		);
+		assert_eq!(
+			selector.select(&info, HeadersToRelay::All, None, &finality_proofs_buf),
+			Some(JustifiedHeader {
+				header: TestSourceHeader(false, 9, 9),
+				proof: TestFinalityProof(9)
+			})
+		);
+
+		// when only free headers needs to be relayed and there are no free headers
+		let finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline>::new(vec![
+			TestFinalityProof(7),
+			TestFinalityProof(9),
+		]);
+		let selector = JustifiedHeaderSelector::None(vec![
+			TestSourceHeader(false, 8, 8),
+			TestSourceHeader(false, 9, 9),
+			TestSourceHeader(false, 10, 10),
+		]);
+		assert_eq!(
+			selector.select(&info, HeadersToRelay::Free, Some(7), &finality_proofs_buf),
+			None,
+		);
+
+		// when only free headers needs to be relayed, mandatory header may be selected
+		let finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline>::new(vec![
+			TestFinalityProof(6),
+			TestFinalityProof(9),
+		]);
+		let selector = JustifiedHeaderSelector::None(vec![
+			TestSourceHeader(false, 8, 8),
+			TestSourceHeader(true, 9, 9),
+			TestSourceHeader(false, 10, 10),
+		]);
+		assert_eq!(
+			selector.select(&info, HeadersToRelay::Free, Some(7), &finality_proofs_buf),
+			Some(JustifiedHeader {
+				header: TestSourceHeader(true, 9, 9),
+				proof: TestFinalityProof(9)
+			})
+		);
+
+		// when only free headers needs to be relayed and there is free header
+		let finality_proofs_buf = FinalityProofsBuf::<TestFinalitySyncPipeline>::new(vec![
+			TestFinalityProof(7),
+			TestFinalityProof(9),
+			TestFinalityProof(14),
+		]);
+		let selector = JustifiedHeaderSelector::None(vec![
+			TestSourceHeader(false, 7, 7),
+			TestSourceHeader(false, 10, 10),
+			TestSourceHeader(false, 14, 14),
+		]);
+		assert_eq!(
+			selector.select(&info, HeadersToRelay::Free, Some(7), &finality_proofs_buf),
+			Some(JustifiedHeader {
+				header: TestSourceHeader(false, 14, 14),
+				proof: TestFinalityProof(14)
+			})
+		);
+	}
+}

pezbridges/relays/finality/src/lib.rs

@@ -0,0 +1,93 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! This crate has single entrypoint to run synchronization loop that is built around finality
+//! proofs, as opposed to headers synchronization loop, which is built around headers. The headers
+//! are still submitted to the target node, but are treated as auxiliary data as we are not trying
+//! to submit all source headers to the target node.
+
+pub use crate::{
+	base::{FinalityPipeline, SourceClientBase},
+	finality_loop::{
+		metrics_prefix, run, FinalitySyncParams, HeadersToRelay, SourceClient, TargetClient,
+	},
+	finality_proofs::{FinalityProofsBuf, FinalityProofsStream},
+	sync_loop_metrics::SyncLoopMetrics,
+};
+
+use bp_header_pez_chain::ConsensusLogReader;
+use relay_utils::{FailedClient, MaybeConnectionError};
+use std::fmt::Debug;
+
+mod base;
+mod finality_loop;
+mod finality_proofs;
+mod headers;
+mod mock;
+mod sync_loop_metrics;
+
+/// Finality proofs synchronization pipeline.
+pub trait FinalitySyncPipeline: FinalityPipeline {
+	/// A reader that can extract the consensus log from the header digest and interpret it.
+	type ConsensusLogReader: ConsensusLogReader;
+	/// Type of header that we're syncing.
+	type Header: SourceHeader<Self::Hash, Self::Number, Self::ConsensusLogReader>;
+}
+
+/// Header that we're receiving from source node.
+pub trait SourceHeader<Hash, Number, Reader>: Clone + Debug + PartialEq + Send + Sync {
+	/// Returns hash of header.
+	fn hash(&self) -> Hash;
+	/// Returns number of header.
+	fn number(&self) -> Number;
+	/// Returns true if this header needs to be submitted to target node.
+	fn is_mandatory(&self) -> bool;
+}
+
+/// Error that may happen inside finality synchronization loop.
+#[derive(Debug)]
+enum Error<P: FinalitySyncPipeline, SourceError, TargetError> {
+	/// Source client request has failed with given error.
+	Source(SourceError),
+	/// Target client request has failed with given error.
+	Target(TargetError),
+	/// Finality proof for mandatory header is missing from the source node.
+	MissingMandatoryFinalityProof(P::Number),
+	/// `submit_finality_proof` transaction failed
+	ProofSubmissionTxFailed {
+		#[allow(dead_code)]
+		submitted_number: P::Number,
+		#[allow(dead_code)]
+		best_number_at_target: P::Number,
+	},
+	/// `submit_finality_proof` transaction lost
+	ProofSubmissionTxLost,
+}
+
+impl<P, SourceError, TargetError> Error<P, SourceError, TargetError>
+where
+	P: FinalitySyncPipeline,
+	SourceError: MaybeConnectionError,
+	TargetError: MaybeConnectionError,
+{
+	fn fail_if_connection_error(&self) -> Result<(), FailedClient> {
+		match *self {
+			Error::Source(ref error) if error.is_connection_error() => Err(FailedClient::Source),
+			Error::Target(ref error) if error.is_connection_error() => Err(FailedClient::Target),
+			_ => Ok(()),
+		}
+	}
+}

pezbridges/relays/finality/src/mock.rs

@@ -0,0 +1,218 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Tests for finality synchronization loop.
+
+#![cfg(test)]
+
+use crate::{
+	base::SourceClientBase,
+	finality_loop::{SourceClient, TargetClient},
+	FinalityPipeline, FinalitySyncPipeline, SourceHeader,
+};
+
+use async_trait::async_trait;
+use bp_header_pez_chain::{FinalityProof, GrandpaConsensusLogReader};
+use futures::{Stream, StreamExt};
+use parking_lot::Mutex;
+use relay_utils::{
+	relay_loop::Client as RelayClient, HeaderId, MaybeConnectionError, TrackedTransactionStatus,
+	TransactionTracker,
+};
+use std::{collections::HashMap, pin::Pin, sync::Arc};
+
+type IsMandatory = bool;
+pub type TestNumber = u64;
+type TestHash = u64;
+
+#[derive(Clone, Debug)]
+pub struct TestTransactionTracker(pub TrackedTransactionStatus<HeaderId<TestHash, TestNumber>>);
+
+impl Default for TestTransactionTracker {
+	fn default() -> TestTransactionTracker {
+		TestTransactionTracker(TrackedTransactionStatus::Finalized(Default::default()))
+	}
+}
+
+#[async_trait]
+impl TransactionTracker for TestTransactionTracker {
+	type HeaderId = HeaderId<TestHash, TestNumber>;
+
+	async fn wait(self) -> TrackedTransactionStatus<HeaderId<TestHash, TestNumber>> {
+		self.0
+	}
+}
+
+#[derive(Debug, Clone)]
+pub enum TestError {
+	NonConnection,
+}
+
+impl MaybeConnectionError for TestError {
+	fn is_connection_error(&self) -> bool {
+		false
+	}
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct TestFinalitySyncPipeline;
+
+impl FinalityPipeline for TestFinalitySyncPipeline {
+	const SOURCE_NAME: &'static str = "TestSource";
+	const TARGET_NAME: &'static str = "TestTarget";
+
+	type Hash = TestHash;
+	type Number = TestNumber;
+	type FinalityProof = TestFinalityProof;
+}
+
+impl FinalitySyncPipeline for TestFinalitySyncPipeline {
+	type ConsensusLogReader = GrandpaConsensusLogReader<TestNumber>;
+	type Header = TestSourceHeader;
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TestSourceHeader(pub IsMandatory, pub TestNumber, pub TestHash);
+
+impl SourceHeader<TestHash, TestNumber, GrandpaConsensusLogReader<TestNumber>>
+	for TestSourceHeader
+{
+	fn hash(&self) -> TestHash {
+		self.2
+	}
+
+	fn number(&self) -> TestNumber {
+		self.1
+	}
+
+	fn is_mandatory(&self) -> bool {
+		self.0
+	}
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TestFinalityProof(pub TestNumber);
+
+impl FinalityProof<TestHash, TestNumber> for TestFinalityProof {
+	fn target_header_hash(&self) -> TestHash {
+		Default::default()
+	}
+
+	fn target_header_number(&self) -> TestNumber {
+		self.0
+	}
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct ClientsData {
+	pub source_best_block_number: TestNumber,
+	pub source_headers: HashMap<TestNumber, (TestSourceHeader, Option<TestFinalityProof>)>,
+	pub source_proofs: Vec<TestFinalityProof>,
+
+	pub target_best_block_id: HeaderId<TestHash, TestNumber>,
+	pub target_headers: Vec<(TestSourceHeader, TestFinalityProof)>,
+	pub target_transaction_tracker: TestTransactionTracker,
+}
+
+#[derive(Clone)]
+pub struct TestSourceClient {
+	pub on_method_call: Arc<dyn Fn(&mut ClientsData) + Send + Sync>,
+	pub data: Arc<Mutex<ClientsData>>,
+}
+
+#[async_trait]
+impl RelayClient for TestSourceClient {
+	type Error = TestError;
+
+	async fn reconnect(&mut self) -> Result<(), TestError> {
+		unreachable!()
+	}
+}
+
+#[async_trait]
+impl SourceClientBase<TestFinalitySyncPipeline> for TestSourceClient {
+	type FinalityProofsStream = Pin<Box<dyn Stream<Item = TestFinalityProof> + 'static + Send>>;
+
+	async fn finality_proofs(&self) -> Result<Self::FinalityProofsStream, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(&mut data);
+		Ok(futures::stream::iter(data.source_proofs.clone()).boxed())
+	}
+}
+
+#[async_trait]
+impl SourceClient<TestFinalitySyncPipeline> for TestSourceClient {
+	async fn best_finalized_block_number(&self) -> Result<TestNumber, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(&mut data);
+		Ok(data.source_best_block_number)
+	}
+
+	async fn header_and_finality_proof(
+		&self,
+		number: TestNumber,
+	) -> Result<(TestSourceHeader, Option<TestFinalityProof>), TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(&mut data);
+		data.source_headers.get(&number).cloned().ok_or(TestError::NonConnection)
+	}
+}
+
+#[derive(Clone)]
+pub struct TestTargetClient {
+	pub on_method_call: Arc<dyn Fn(&mut ClientsData) + Send + Sync>,
+	pub data: Arc<Mutex<ClientsData>>,
+}
+
+#[async_trait]
+impl RelayClient for TestTargetClient {
+	type Error = TestError;
+
+	async fn reconnect(&mut self) -> Result<(), TestError> {
+		unreachable!()
+	}
+}
+
+#[async_trait]
+impl TargetClient<TestFinalitySyncPipeline> for TestTargetClient {
+	type TransactionTracker = TestTransactionTracker;
+
+	async fn best_finalized_source_block_id(
+		&self,
+	) -> Result<HeaderId<TestHash, TestNumber>, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(&mut data);
+		Ok(data.target_best_block_id)
+	}
+
+	async fn free_source_headers_interval(&self) -> Result<Option<TestNumber>, TestError> {
+		Ok(Some(3))
+	}
+
+	async fn submit_finality_proof(
+		&self,
+		header: TestSourceHeader,
+		proof: TestFinalityProof,
+		_is_free_execution_expected: bool,
+	) -> Result<TestTransactionTracker, TestError> {
+		let mut data = self.data.lock();
+		(self.on_method_call)(&mut data);
+		data.target_best_block_id = HeaderId(header.number(), header.hash());
+		data.target_headers.push((header, proof));
+		(self.on_method_call)(&mut data);
+		Ok(data.target_transaction_tracker.clone())
+	}
+}

pezbridges/relays/finality/src/sync_loop_metrics.rs

@@ -0,0 +1,95 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Metrics for headers synchronization relay loop.
+
+use relay_utils::{
+	metrics::{metric_name, register, IntGauge, Metric, PrometheusError, Registry},
+	UniqueSaturatedInto,
+};
+
+/// Headers sync metrics.
+#[derive(Clone)]
+pub struct SyncLoopMetrics {
+	/// Best syncing header at the source.
+	best_source_block_number: IntGauge,
+	/// Best syncing header at the target.
+	best_target_block_number: IntGauge,
+	/// Flag that has `0` value when best source headers at the source node and at-target-chain
+	/// are matching and `1` otherwise.
+	using_different_forks: IntGauge,
+}
+
+impl SyncLoopMetrics {
+	/// Create and register headers loop metrics.
+	pub fn new(
+		prefix: Option<&str>,
+		at_source_chain_label: &str,
+		at_target_chain_label: &str,
+	) -> Result<Self, PrometheusError> {
+		Ok(SyncLoopMetrics {
+			best_source_block_number: IntGauge::new(
+				metric_name(prefix, &format!("best_{at_source_chain_label}_block_number")),
+				format!("Best block number at the {at_source_chain_label}"),
+			)?,
+			best_target_block_number: IntGauge::new(
+				metric_name(prefix, &format!("best_{at_target_chain_label}_block_number")),
+				format!("Best block number at the {at_target_chain_label}"),
+			)?,
+			using_different_forks: IntGauge::new(
+				metric_name(prefix, &format!("is_{at_source_chain_label}_and_{at_target_chain_label}_using_different_forks")),
+				"Whether the best finalized source block at target node is different (value 1) from the \
+				corresponding block at the source node",
+			)?,
+		})
+	}
+
+	/// Returns current value of the using-same-fork flag.
+	#[cfg(test)]
+	pub(crate) fn is_using_same_fork(&self) -> bool {
+		self.using_different_forks.get() == 0
+	}
+
+	/// Update best block number at source.
+	pub fn update_best_block_at_source<Number: UniqueSaturatedInto<u64>>(
+		&self,
+		source_best_number: Number,
+	) {
+		self.best_source_block_number.set(source_best_number.unique_saturated_into());
+	}
+
+	/// Update best block number at target.
+	pub fn update_best_block_at_target<Number: UniqueSaturatedInto<u64>>(
+		&self,
+		target_best_number: Number,
+	) {
+		self.best_target_block_number.set(target_best_number.unique_saturated_into());
+	}
+
+	/// Update using-same-fork flag.
+	pub fn update_using_same_fork(&self, using_same_fork: bool) {
+		self.using_different_forks.set((!using_same_fork).into())
+	}
+}
+
+impl Metric for SyncLoopMetrics {
+	fn register(&self, registry: &Registry) -> Result<(), PrometheusError> {
+		register(self.best_source_block_number.clone(), registry)?;
+		register(self.best_target_block_number.clone(), registry)?;
+		register(self.using_different_forks.clone(), registry)?;
+		Ok(())
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/Cargo.toml

@@ -0,0 +1,88 @@
+[package]
+name = "bizinikiwi-relay-helper"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+repository.workspace = true
+publish = false
+description = "Bizinikiwi utility: relay helper"
+documentation = "https://docs.rs/bizinikiwi-relay-helper"
+homepage = { workspace = true }
+
+[lints]
+workspace = true
+
+[dependencies]
+anyhow = { workspace = true }
+async-std = { workspace = true }
+async-trait = { workspace = true }
+clap = { workspace = true, features = ["derive"] }
+codec = { workspace = true, default-features = true }
+futures = { workspace = true }
+hex = { workspace = true, default-features = true }
+num-traits = { workspace = true, default-features = true }
+rbtag = { workspace = true }
+strum = { features = ["derive"], workspace = true, default-features = true }
+thiserror = { workspace = true }
+tracing = { workspace = true }
+
+# Bridge dependencies
+bp-header-pez-chain = { workspace = true, default-features = true }
+bp-pezkuwi-core = { workspace = true, default-features = true }
+bp-relayers = { workspace = true, default-features = true }
+bp-teyrchains = { workspace = true, default-features = true }
+
+pez-equivocation-detector = { workspace = true }
+pez-finality-relay = { workspace = true }
+pez-messages-relay = { workspace = true }
+relay-bizinikiwi-client = { workspace = true }
+relay-utils = { workspace = true }
+teyrchains-relay = { workspace = true }
+
+pezpallet-bridge-grandpa = { workspace = true, default-features = true }
+pezpallet-bridge-messages = { workspace = true, default-features = true }
+pezpallet-bridge-teyrchains = { workspace = true, default-features = true }
+
+bp-messages = { workspace = true, default-features = true }
+pezbp-runtime = { workspace = true, default-features = true }
+
+# Bizinikiwi Dependencies
+pezframe-support = { workspace = true, default-features = true }
+pezframe-system = { workspace = true, default-features = true }
+pezpallet-balances = { workspace = true, default-features = true }
+pezpallet-grandpa = { workspace = true, default-features = true }
+pezsp-consensus-grandpa = { workspace = true, default-features = true }
+pezsp-core = { workspace = true, default-features = true }
+pezsp-runtime = { workspace = true, default-features = true }
+pezsp-trie = { workspace = true }
+
+[dev-dependencies]
+relay-bizinikiwi-client = { features = ["test-helpers"], workspace = true }
+scale-info = { features = ["derive"], workspace = true }
+
+[features]
+runtime-benchmarks = [
+  "bp-header-pez-chain/runtime-benchmarks",
+  "bp-messages/runtime-benchmarks",
+  "bp-pezkuwi-core/runtime-benchmarks",
+  "bp-relayers/runtime-benchmarks",
+  "pezbp-runtime/runtime-benchmarks",
+  "bp-teyrchains/runtime-benchmarks",
+  "pez-equivocation-detector/runtime-benchmarks",
+  "pez-finality-relay/runtime-benchmarks",
+  "pezframe-support/runtime-benchmarks",
+  "pezframe-system/runtime-benchmarks",
+  "pez-messages-relay/runtime-benchmarks",
+  "pezpallet-balances/runtime-benchmarks",
+  "pezpallet-bridge-grandpa/runtime-benchmarks",
+  "pezpallet-bridge-messages/runtime-benchmarks",
+  "pezpallet-bridge-teyrchains/runtime-benchmarks",
+  "pezpallet-grandpa/runtime-benchmarks",
+  "relay-bizinikiwi-client/runtime-benchmarks",
+  "relay-utils/runtime-benchmarks",
+  "pezsp-consensus-grandpa/runtime-benchmarks",
+  "pezsp-runtime/runtime-benchmarks",
+  "pezsp-trie/runtime-benchmarks",
+  "teyrchains-relay/runtime-benchmarks",
+]

pezbridges/relays/lib-bizinikiwi-relay/src/cli/bridge.rs

@@ -0,0 +1,114 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Basic traits for exposing bridges in the CLI.
+
+use crate::{
+	equivocation::BizinikiwiEquivocationDetectionPipeline,
+	finality::BizinikiwiFinalitySyncPipeline,
+	messages::{MessagesRelayLimits, BizinikiwiMessageLane},
+	teyrchains::BizinikiwiTeyrchainsPipeline,
+};
+use bp_teyrchains::{RelayBlockHash, RelayBlockHasher, RelayBlockNumber};
+use relay_bizinikiwi_client::{
+	Chain, ChainWithRuntimeVersion, ChainWithTransactions, RelayChain, Teyrchain,
+};
+
+/// Minimal bridge representation that can be used from the CLI.
+/// It connects a source chain to a target chain.
+pub trait CliBridgeBase: Sized {
+	/// The source chain.
+	type Source: Chain + ChainWithRuntimeVersion;
+	/// The target chain.
+	type Target: ChainWithTransactions + ChainWithRuntimeVersion;
+}
+
+/// Bridge representation that can be used from the CLI for relaying headers
+/// from a relay chain to a relay chain.
+pub trait RelayToRelayHeadersCliBridge: CliBridgeBase {
+	/// Finality proofs synchronization pipeline.
+	type Finality: BizinikiwiFinalitySyncPipeline<
+		SourceChain = Self::Source,
+		TargetChain = Self::Target,
+	>;
+}
+
+/// Convenience trait that adds bounds to `CliBridgeBase`.
+pub trait RelayToRelayEquivocationDetectionCliBridgeBase: CliBridgeBase {
+	/// The source chain with extra bounds.
+	type BoundedSource: ChainWithTransactions;
+}
+
+impl<T> RelayToRelayEquivocationDetectionCliBridgeBase for T
+where
+	T: CliBridgeBase,
+	T::Source: ChainWithTransactions,
+{
+	type BoundedSource = T::Source;
+}
+
+/// Bridge representation that can be used from the CLI for detecting equivocations
+/// in the headers synchronized from a relay chain to a relay chain.
+pub trait RelayToRelayEquivocationDetectionCliBridge:
+	RelayToRelayEquivocationDetectionCliBridgeBase
+{
+	/// Equivocation detection pipeline.
+	type Equivocation: BizinikiwiEquivocationDetectionPipeline<
+		SourceChain = Self::Source,
+		TargetChain = Self::Target,
+	>;
+}
+
+/// Bridge representation that can be used from the CLI for relaying headers
+/// from a teyrchain to a relay chain.
+pub trait TeyrchainToRelayHeadersCliBridge: CliBridgeBase
+where
+	Self::Source: Teyrchain,
+{
+	/// The `CliBridgeBase` type represents the teyrchain in this situation.
+	/// We need to add an extra type for the relay chain.
+	type SourceRelay: Chain<BlockNumber = RelayBlockNumber, Hash = RelayBlockHash, Hasher = RelayBlockHasher>
+		+ ChainWithRuntimeVersion
+		+ RelayChain;
+	/// Finality proofs synchronization pipeline (source teyrchain -> target).
+	type TeyrchainFinality: BizinikiwiTeyrchainsPipeline<
+		SourceRelayChain = Self::SourceRelay,
+		SourceTeyrchain = Self::Source,
+		TargetChain = Self::Target,
+	>;
+	/// Finality proofs synchronization pipeline (source relay chain -> target).
+	type RelayFinality: BizinikiwiFinalitySyncPipeline<
+		SourceChain = Self::SourceRelay,
+		TargetChain = Self::Target,
+	>;
+}
+
+/// Bridge representation that can be used from the CLI for relaying messages.
+pub trait MessagesCliBridge: CliBridgeBase {
+	/// The Source -> Destination messages synchronization pipeline.
+	type MessagesLane: BizinikiwiMessageLane<SourceChain = Self::Source, TargetChain = Self::Target>;
+
+	/// Optional messages delivery transaction limits that the messages relay is going
+	/// to use. If it returns `None`, limits are estimated using `TransactionPayment` API
+	/// at the target chain.
+	fn maybe_messages_limits() -> Option<MessagesRelayLimits> {
+		None
+	}
+}
+
+/// An alias for lane identifier type.
+pub type MessagesLaneIdOf<B> =
+	<<B as MessagesCliBridge>::MessagesLane as BizinikiwiMessageLane>::LaneId;

pezbridges/relays/lib-bizinikiwi-relay/src/cli/chain_schema.rs

@@ -0,0 +1,245 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Primitives related to chain CLI options.
+
+use clap::Parser;
+use relay_bizinikiwi_client::{AccountKeyPairOf, ChainWithTransactions};
+use strum::{EnumString, VariantNames};
+
+use relay_bizinikiwi_client::{ChainRuntimeVersion, ChainWithRuntimeVersion, SimpleRuntimeVersion};
+
+use crate::TransactionParams;
+
+#[doc = "Runtime version params."]
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Parser, EnumString, VariantNames)]
+pub enum RuntimeVersionType {
+	/// Auto query version from chain
+	Auto,
+	/// Custom `spec_version` and `transaction_version`
+	Custom,
+	/// Read version from bundle dependencies directly.
+	Bundle,
+}
+
+/// Create chain-specific set of runtime version parameters.
+#[macro_export]
+macro_rules! declare_chain_runtime_version_params_cli_schema {
+	($chain:ident, $chain_prefix:ident) => {
+		pezbp_runtime::paste::item! {
+			#[doc = $chain " runtime version params."]
+			#[derive(Debug, PartialEq, Eq, Clone, Copy, Parser)]
+			pub struct [<$chain RuntimeVersionParams>] {
+				#[doc = "The type of runtime version for chain " $chain]
+				#[arg(long, default_value = "Bundle")]
+				pub [<$chain_prefix _version_mode>]: RuntimeVersionType,
+				#[doc = "The custom sepc_version for chain " $chain]
+				#[arg(long)]
+				pub [<$chain_prefix _spec_version>]: Option<u32>,
+				#[doc = "The custom transaction_version for chain " $chain]
+				#[arg(long)]
+				pub [<$chain_prefix _transaction_version>]: Option<u32>,
+			}
+
+			impl [<$chain RuntimeVersionParams>] {
+				/// Converts self into `ChainRuntimeVersion`.
+				pub fn into_runtime_version(
+					self,
+					bundle_runtime_version: Option<SimpleRuntimeVersion>,
+				) -> anyhow::Result<ChainRuntimeVersion> {
+					Ok(match self.[<$chain_prefix _version_mode>] {
+						RuntimeVersionType::Auto => ChainRuntimeVersion::Auto,
+						RuntimeVersionType::Custom => {
+							let custom_spec_version = self.[<$chain_prefix _spec_version>]
+								.ok_or_else(|| anyhow::Error::msg(format!("The {}-spec-version is required when choose custom mode", stringify!($chain_prefix))))?;
+							let custom_transaction_version = self.[<$chain_prefix _transaction_version>]
+								.ok_or_else(|| anyhow::Error::msg(format!("The {}-transaction-version is required when choose custom mode", stringify!($chain_prefix))))?;
+							ChainRuntimeVersion::Custom(
+								SimpleRuntimeVersion {
+									spec_version: custom_spec_version,
+									transaction_version: custom_transaction_version
+								}
+							)
+						},
+						RuntimeVersionType::Bundle => match bundle_runtime_version {
+							Some(runtime_version) => ChainRuntimeVersion::Custom(runtime_version),
+							None => {
+								return Err(anyhow::format_err!("Cannot use bundled runtime version of {}: it is not known to the relay", stringify!($chain_prefix)));
+							}
+						},
+					})
+				}
+			}
+		}
+	};
+}
+
+/// Create chain-specific set of runtime version parameters.
+#[macro_export]
+macro_rules! declare_chain_connection_params_cli_schema {
+	($chain:ident, $chain_prefix:ident) => {
+		pezbp_runtime::paste::item! {
+			// TODO: https://github.com/pezkuwichain/kurdistan-sdk/issues/86
+			// remove all obsolete arguments (separate URI components)
+
+			#[doc = $chain " connection params."]
+			#[derive(Debug, PartialEq, Eq, Clone, Parser)]
+			pub struct [<$chain ConnectionParams>] {
+				#[doc = "WS endpoint of " $chain ": full URI."]
+				#[arg(long)]
+				pub [<$chain_prefix _uri>]: String,
+				#[doc = "Custom runtime version"]
+				#[command(flatten)]
+				pub [<$chain_prefix _runtime_version>]: [<$chain RuntimeVersionParams>],
+			}
+
+			impl [<$chain ConnectionParams>] {
+				/// Convert connection params into Bizinikiwi client.
+				#[allow(dead_code)]
+				pub async fn into_client<Chain: ChainWithRuntimeVersion>(
+					self,
+				) -> anyhow::Result<$crate::cli::DefaultClient<Chain>> {
+					let chain_runtime_version = self
+						.[<$chain_prefix _runtime_version>]
+						.into_runtime_version(Chain::RUNTIME_VERSION)?;
+					Ok(relay_bizinikiwi_client::new(relay_bizinikiwi_client::ConnectionParams {
+						uri: self.[<$chain_prefix _uri>],
+						chain_runtime_version,
+					})
+					.await
+					)
+				}
+			}
+		}
+	};
+}
+
+/// Create chain-specific set of signing parameters.
+#[macro_export]
+macro_rules! declare_chain_signing_params_cli_schema {
+	($chain:ident, $chain_prefix:ident) => {
+		pezbp_runtime::paste::item! {
+			#[doc = $chain " signing params."]
+			#[derive(Debug, PartialEq, Eq, Clone, Parser)]
+			pub struct [<$chain SigningParams>] {
+				#[doc = "The SURI of secret key to use when transactions are submitted to the " $chain " node."]
+				#[arg(long)]
+				pub [<$chain_prefix _signer>]: Option<String>,
+				#[doc = "The password for the SURI of secret key to use when transactions are submitted to the " $chain " node."]
+				#[arg(long)]
+				pub [<$chain_prefix _signer_password>]: Option<String>,
+
+				#[doc = "Path to the file, that contains SURI of secret key to use when transactions are submitted to the " $chain " node. Can be overridden with " $chain_prefix "_signer option."]
+				#[arg(long)]
+				pub [<$chain_prefix _signer_file>]: Option<std::path::PathBuf>,
+				#[doc = "Path to the file, that password for the SURI of secret key to use when transactions are submitted to the " $chain " node. Can be overridden with " $chain_prefix "_signer_password option."]
+				#[arg(long)]
+				pub [<$chain_prefix _signer_password_file>]: Option<std::path::PathBuf>,
+
+				#[doc = "Transactions mortality period, in blocks. MUST be a power of two in [4; 65536] range. MAY NOT be larger than `BlockHashCount` parameter of the chain system module."]
+				#[arg(long)]
+				pub [<$chain_prefix _transactions_mortality>]: Option<u32>,
+			}
+
+			impl [<$chain SigningParams>] {
+				/// Return transactions mortality.
+				#[allow(dead_code)]
+				pub fn transactions_mortality(&self) -> anyhow::Result<Option<u32>> {
+					self.[<$chain_prefix _transactions_mortality>]
+						.map(|transactions_mortality| {
+							if !(4..=65536).contains(&transactions_mortality)
+								|| !transactions_mortality.is_power_of_two()
+							{
+								Err(anyhow::format_err!(
+									"Transactions mortality {} is not a power of two in a [4; 65536] range",
+									transactions_mortality,
+								))
+							} else {
+								Ok(transactions_mortality)
+							}
+						})
+						.transpose()
+				}
+
+				/// Parse signing params into chain-specific KeyPair.
+				#[allow(dead_code)]
+				pub fn to_keypair<Chain: ChainWithTransactions>(&self) -> anyhow::Result<AccountKeyPairOf<Chain>> {
+					let suri = match (self.[<$chain_prefix _signer>].as_ref(), self.[<$chain_prefix _signer_file>].as_ref()) {
+						(Some(suri), _) => suri.to_owned(),
+						(None, Some(suri_file)) => std::fs::read_to_string(suri_file)
+							.map_err(|err| anyhow::format_err!(
+								"Failed to read SURI from file {:?}: {}",
+								suri_file,
+								err,
+							))?,
+						(None, None) => return Err(anyhow::format_err!(
+							"One of options must be specified: '{}' or '{}'",
+							stringify!([<$chain_prefix _signer>]),
+							stringify!([<$chain_prefix _signer_file>]),
+						)),
+					};
+
+					let suri_password = match (
+						self.[<$chain_prefix _signer_password>].as_ref(),
+						self.[<$chain_prefix _signer_password_file>].as_ref(),
+					) {
+						(Some(suri_password), _) => Some(suri_password.to_owned()),
+						(None, Some(suri_password_file)) => std::fs::read_to_string(suri_password_file)
+							.map(Some)
+							.map_err(|err| anyhow::format_err!(
+								"Failed to read SURI password from file {:?}: {}",
+								suri_password_file,
+								err,
+							))?,
+						_ => None,
+					};
+
+					use pezsp_core::crypto::Pair;
+
+					AccountKeyPairOf::<Chain>::from_string(
+						&suri,
+						suri_password.as_deref()
+					).map_err(|e| anyhow::format_err!("{:?}", e))
+				}
+
+				/// Return transaction parameters.
+				#[allow(dead_code)]
+				pub fn transaction_params<Chain: ChainWithTransactions>(
+					&self,
+				) -> anyhow::Result<TransactionParams<AccountKeyPairOf<Chain>>> {
+					Ok(TransactionParams {
+						mortality: self.transactions_mortality()?,
+						signer: self.to_keypair::<Chain>()?,
+					})
+				}
+			}
+		}
+	};
+}
+
+/// Create chain-specific set of configuration objects: connection parameters,
+/// signing parameters and bridge initialization parameters.
+#[macro_export]
+macro_rules! declare_chain_cli_schema {
+	($chain:ident, $chain_prefix:ident) => {
+		$crate::declare_chain_runtime_version_params_cli_schema!($chain, $chain_prefix);
+		$crate::declare_chain_connection_params_cli_schema!($chain, $chain_prefix);
+		$crate::declare_chain_signing_params_cli_schema!($chain, $chain_prefix);
+	};
+}
+
+declare_chain_cli_schema!(Source, source);
+declare_chain_cli_schema!(Target, target);

pezbridges/relays/lib-bizinikiwi-relay/src/cli/detect_equivocations.rs

@@ -0,0 +1,65 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Primitives for exposing the equivocation detection functionality in the CLI.
+
+use crate::{
+	cli::{bridge::*, chain_schema::*, PrometheusParams},
+	equivocation,
+	equivocation::BizinikiwiEquivocationDetectionPipeline,
+};
+
+use async_trait::async_trait;
+use clap::Parser;
+use relay_bizinikiwi_client::{ChainWithTransactions, Client};
+
+/// Start equivocation detection loop.
+#[derive(Parser)]
+pub struct DetectEquivocationsParams {
+	#[command(flatten)]
+	source: SourceConnectionParams,
+	#[command(flatten)]
+	source_sign: SourceSigningParams,
+	#[command(flatten)]
+	target: TargetConnectionParams,
+	#[command(flatten)]
+	prometheus_params: PrometheusParams,
+}
+
+/// Trait used for starting the equivocation detection loop between 2 chains.
+#[async_trait]
+pub trait EquivocationsDetector: RelayToRelayEquivocationDetectionCliBridge
+where
+	Self::Source: ChainWithTransactions,
+{
+	/// Start the equivocation detection loop.
+	async fn start(data: DetectEquivocationsParams) -> anyhow::Result<()> {
+		let source_client = data.source.into_client::<Self::Source>().await?;
+		Self::Equivocation::start_relay_guards(
+			&source_client,
+			source_client.can_start_version_guard(),
+		)
+		.await?;
+
+		equivocation::run::<Self::Equivocation>(
+			source_client,
+			data.target.into_client::<Self::Target>().await?,
+			data.source_sign.transaction_params::<Self::Source>()?,
+			data.prometheus_params.into_metrics_params()?,
+		)
+		.await
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/cli/init_bridge.rs

@@ -0,0 +1,85 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Primitives for exposing the bridge initialization functionality in the CLI.
+
+use async_trait::async_trait;
+use codec::Encode;
+
+use crate::{
+	cli::{bridge::CliBridgeBase, chain_schema::*},
+	finality_base::engine::Engine,
+};
+use pezbp_runtime::Chain as ChainBase;
+use clap::Parser;
+use relay_bizinikiwi_client::{AccountKeyPairOf, Chain, UnsignedTransaction};
+use pezsp_core::Pair;
+
+/// Bridge initialization params.
+#[derive(Parser)]
+pub struct InitBridgeParams {
+	#[command(flatten)]
+	source: SourceConnectionParams,
+	#[command(flatten)]
+	target: TargetConnectionParams,
+	#[command(flatten)]
+	target_sign: TargetSigningParams,
+	/// Generates all required data, but does not submit extrinsic
+	#[arg(long)]
+	dry_run: bool,
+}
+
+/// Trait used for bridge initializing.
+#[async_trait]
+pub trait BridgeInitializer: CliBridgeBase
+where
+	<Self::Target as ChainBase>::AccountId: From<<AccountKeyPairOf<Self::Target> as Pair>::Public>,
+{
+	/// The finality engine used by the source chain.
+	type Engine: Engine<Self::Source>;
+
+	/// Get the encoded call to init the bridge.
+	fn encode_init_bridge(
+		init_data: <Self::Engine as Engine<Self::Source>>::InitializationData,
+	) -> <Self::Target as Chain>::Call;
+
+	/// Initialize the bridge.
+	async fn init_bridge(data: InitBridgeParams) -> anyhow::Result<()> {
+		let source_client = data.source.into_client::<Self::Source>().await?;
+		let target_client = data.target.into_client::<Self::Target>().await?;
+		let target_sign = data.target_sign.to_keypair::<Self::Target>()?;
+		let dry_run = data.dry_run;
+
+		crate::finality::initialize::initialize::<Self::Engine, _, _, _>(
+			source_client,
+			target_client.clone(),
+			target_sign,
+			move |transaction_nonce, initialization_data| {
+				let call = Self::encode_init_bridge(initialization_data);
+				tracing::info!(
+					target: "bridge",
+					hex_string=?format!("0x{}", hex::encode(call.encode())),
+					"Initialize bridge call encoded"
+				);
+				Ok(UnsignedTransaction::new(call.into(), transaction_nonce))
+			},
+			dry_run,
+		)
+		.await;
+
+		Ok(())
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/cli/mod.rs

@@ -0,0 +1,176 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Deal with CLI args of bizinikiwi-to-bizinikiwi relay.
+
+use clap::Parser;
+use rbtag::BuildInfo;
+use pezsp_runtime::traits::TryConvert;
+use std::str::FromStr;
+
+pub mod bridge;
+pub mod chain_schema;
+pub mod detect_equivocations;
+pub mod init_bridge;
+pub mod relay_headers;
+pub mod relay_headers_and_messages;
+pub mod relay_messages;
+pub mod relay_teyrchains;
+
+/// The target that will be used when publishing logs related to this pezpallet.
+pub const LOG_TARGET: &str = "bridge";
+
+/// Default Bizinikiwi client type that we are using. We'll use it all over the glue CLI code
+/// to avoid multiple level generic arguments and constraints. We still allow usage of other
+/// clients in the **core logic code**.
+pub type DefaultClient<C> = relay_bizinikiwi_client::RpcWithCachingClient<C>;
+
+/// Lane id.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct HexLaneId(Vec<u8>);
+
+impl<T: TryFrom<Vec<u8>>> TryConvert<HexLaneId, T> for HexLaneId {
+	fn try_convert(lane_id: HexLaneId) -> Result<T, HexLaneId> {
+		T::try_from(lane_id.0.clone()).map_err(|_| lane_id)
+	}
+}
+
+impl FromStr for HexLaneId {
+	type Err = hex::FromHexError;
+
+	fn from_str(s: &str) -> Result<Self, Self::Err> {
+		hex::decode(s).map(Self)
+	}
+}
+
+/// Prometheus metrics params.
+#[derive(Clone, Debug, PartialEq, Parser)]
+pub struct PrometheusParams {
+	/// Do not expose a Prometheus metric endpoint.
+	#[arg(long)]
+	pub no_prometheus: bool,
+	/// Expose Prometheus endpoint at given interface.
+	#[arg(long, default_value = "127.0.0.1")]
+	pub prometheus_host: String,
+	/// Expose Prometheus endpoint at given port.
+	#[arg(long, default_value = "9616")]
+	pub prometheus_port: u16,
+}
+
+/// Struct to get git commit info and build time.
+#[derive(BuildInfo)]
+struct BizinikiwiRelayBuildInfo;
+
+impl BizinikiwiRelayBuildInfo {
+	/// Get git commit in form `<short-sha-(clean|dirty)>`.
+	pub fn get_git_commit() -> String {
+		// on gitlab we use images without git installed, so we can't use `rbtag` there
+		// locally we don't have `CI_*` env variables, so we can't rely on them
+		// => we are using `CI_*` env variables or else `rbtag`
+		let maybe_sha_from_ci = option_env!("CI_COMMIT_SHORT_SHA");
+		maybe_sha_from_ci
+			.map(|short_sha| {
+				// we assume that on CI the copy is always clean
+				format!("{short_sha}-clean")
+			})
+			.unwrap_or_else(|| BizinikiwiRelayBuildInfo.get_build_commit().into())
+	}
+}
+
+impl PrometheusParams {
+	/// Tries to convert CLI metrics params into metrics params, used by the relay.
+	pub fn into_metrics_params(self) -> anyhow::Result<relay_utils::metrics::MetricsParams> {
+		let metrics_address = if !self.no_prometheus {
+			Some(relay_utils::metrics::MetricsAddress {
+				host: self.prometheus_host,
+				port: self.prometheus_port,
+			})
+		} else {
+			None
+		};
+
+		let relay_version = relay_utils::initialize::RELAYER_VERSION
+			.lock()
+			.clone()
+			.unwrap_or_else(|| "unknown".to_string());
+		let relay_commit = BizinikiwiRelayBuildInfo::get_git_commit();
+		relay_utils::metrics::MetricsParams::new(metrics_address, relay_version, relay_commit)
+			.map_err(|e| anyhow::format_err!("{:?}", e))
+	}
+}
+
+/// Either explicit or maximal allowed value.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum ExplicitOrMaximal<V> {
+	/// User has explicitly specified argument value.
+	Explicit(V),
+	/// Maximal allowed value for this argument.
+	Maximal,
+}
+
+impl<V: std::str::FromStr> std::str::FromStr for ExplicitOrMaximal<V>
+where
+	V::Err: std::fmt::Debug,
+{
+	type Err = String;
+
+	fn from_str(s: &str) -> Result<Self, Self::Err> {
+		if s.to_lowercase() == "max" {
+			return Ok(ExplicitOrMaximal::Maximal);
+		}
+
+		V::from_str(s)
+			.map(ExplicitOrMaximal::Explicit)
+			.map_err(|e| format!("Failed to parse '{e:?}'. Expected 'max' or explicit value"))
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use bp_messages::{HashedLaneId, LegacyLaneId};
+	use pezsp_core::H256;
+
+	#[test]
+	fn hex_lane_id_from_str_works() {
+		// hash variant
+		assert!(HexLaneId::from_str(
+			"101010101010101010101010101010101010101010101010101010101010101"
+		)
+		.is_err());
+		assert!(HexLaneId::from_str(
+			"00101010101010101010101010101010101010101010101010101010101010101"
+		)
+		.is_err());
+		assert_eq!(
+			HexLaneId::try_convert(
+				HexLaneId::from_str(
+					"0101010101010101010101010101010101010101010101010101010101010101"
+				)
+				.unwrap()
+			),
+			Ok(HashedLaneId::from_inner(H256::from([1u8; 32])))
+		);
+
+		// array variant
+		assert!(HexLaneId::from_str("0000001").is_err());
+		assert!(HexLaneId::from_str("000000001").is_err());
+		assert_eq!(
+			HexLaneId::try_convert(HexLaneId::from_str("00000001").unwrap()),
+			Ok(LegacyLaneId([0, 0, 0, 1]))
+		);
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/cli/relay_headers.rs

@@ -0,0 +1,131 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Primitives for exposing the headers relaying functionality in the CLI.
+
+use async_trait::async_trait;
+use clap::Parser;
+
+use relay_utils::{
+	metrics::{GlobalMetrics, StandaloneMetric},
+	UniqueSaturatedInto,
+};
+
+use crate::{
+	cli::{bridge::*, chain_schema::*, PrometheusParams},
+	finality::BizinikiwiFinalitySyncPipeline,
+	HeadersToRelay,
+};
+use relay_bizinikiwi_client::Client;
+
+/// Chain headers relaying params.
+#[derive(Parser)]
+pub struct RelayHeadersParams {
+	/// If passed, only mandatory headers (headers that are changing the GRANDPA authorities set)
+	/// are relayed.
+	#[arg(long)]
+	only_mandatory_headers: bool,
+	/// If passed, only free headers (mandatory and every Nth header, if configured in runtime)
+	/// are relayed. Overrides `only_mandatory_headers`.
+	#[arg(long)]
+	only_free_headers: bool,
+	#[command(flatten)]
+	source: SourceConnectionParams,
+	#[command(flatten)]
+	target: TargetConnectionParams,
+	#[command(flatten)]
+	target_sign: TargetSigningParams,
+	#[command(flatten)]
+	prometheus_params: PrometheusParams,
+}
+
+/// Single header relaying params.
+#[derive(Parser)]
+pub struct RelayHeaderParams {
+	#[command(flatten)]
+	source: SourceConnectionParams,
+	#[command(flatten)]
+	target: TargetConnectionParams,
+	#[command(flatten)]
+	target_sign: TargetSigningParams,
+	/// Number of the source chain header that we want to relay. It must have a persistent
+	/// storage proof at the [`Self::source`] node, otherwise the command will fail.
+	#[arg(long)]
+	number: u128,
+}
+
+impl RelayHeadersParams {
+	fn headers_to_relay(&self) -> HeadersToRelay {
+		match (self.only_mandatory_headers, self.only_free_headers) {
+			(_, true) => HeadersToRelay::Free,
+			(true, false) => HeadersToRelay::Mandatory,
+			_ => HeadersToRelay::All,
+		}
+	}
+}
+
+/// Trait used for relaying headers between 2 chains.
+#[async_trait]
+pub trait HeadersRelayer: RelayToRelayHeadersCliBridge {
+	/// Relay headers.
+	async fn relay_headers(data: RelayHeadersParams) -> anyhow::Result<()> {
+		let headers_to_relay = data.headers_to_relay();
+		let source_client = data.source.into_client::<Self::Source>().await?;
+		let target_client = data.target.into_client::<Self::Target>().await?;
+		let target_transactions_mortality = data.target_sign.target_transactions_mortality;
+		let target_sign = data.target_sign.to_keypair::<Self::Target>()?;
+
+		let metrics_params: relay_utils::metrics::MetricsParams =
+			data.prometheus_params.into_metrics_params()?;
+		GlobalMetrics::new()?.register_and_spawn(&metrics_params.registry)?;
+
+		let target_transactions_params = crate::TransactionParams {
+			signer: target_sign,
+			mortality: target_transactions_mortality,
+		};
+
+		Self::Finality::start_relay_guards(&target_client, target_client.can_start_version_guard())
+			.await?;
+
+		crate::finality::run::<Self::Finality>(
+			source_client,
+			target_client,
+			headers_to_relay,
+			target_transactions_params,
+			metrics_params,
+		)
+		.await
+	}
+
+	/// Relay single header. No checks are made to ensure that transaction will succeed.
+	async fn relay_header(data: RelayHeaderParams) -> anyhow::Result<()> {
+		let source_client = data.source.into_client::<Self::Source>().await?;
+		let target_client = data.target.into_client::<Self::Target>().await?;
+		let target_transactions_mortality = data.target_sign.target_transactions_mortality;
+		let target_sign = data.target_sign.to_keypair::<Self::Target>()?;
+
+		crate::finality::relay_single_header::<Self::Finality>(
+			source_client,
+			target_client,
+			crate::TransactionParams {
+				signer: target_sign,
+				mortality: target_transactions_mortality,
+			},
+			data.number.unique_saturated_into(),
+		)
+		.await
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/cli/relay_headers_and_messages/mod.rs

@@ -0,0 +1,503 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Complex 2-ways headers+messages relays support.
+//!
+//! To add new complex relay between `ChainA` and `ChainB`, you must:
+//!
+//! 1) ensure that there's a `declare_chain_cli_schema!(...)` for both chains.
+//! 2) add `declare_chain_to_chain_bridge_schema!(...)` or
+//!    `declare_chain_to_teyrchain_bridge_schema` for the bridge.
+//! 3) declare a new struct for the added bridge and implement the `Full2WayBridge` trait for it.
+
+#[macro_use]
+pub mod teyrchain_to_teyrchain;
+#[macro_use]
+pub mod relay_to_relay;
+#[macro_use]
+pub mod relay_to_teyrchain;
+
+use async_trait::async_trait;
+use clap::Parser;
+use std::{fmt::Debug, marker::PhantomData, sync::Arc};
+
+use futures::{FutureExt, TryFutureExt};
+
+use crate::{
+	cli::{
+		bridge::{MessagesCliBridge, MessagesLaneIdOf},
+		DefaultClient, HexLaneId, PrometheusParams,
+	},
+	messages::{MessagesRelayLimits, MessagesRelayParams},
+	on_demand::OnDemandRelay,
+	HeadersToRelay, TaggedAccount, TransactionParams,
+};
+use pezbp_runtime::BalanceOf;
+use relay_bizinikiwi_client::{
+	AccountIdOf, AccountKeyPairOf, Chain, ChainWithBalances, ChainWithMessages,
+	ChainWithRuntimeVersion, ChainWithTransactions,
+};
+use relay_utils::metrics::MetricsParams;
+use pezsp_core::Pair;
+use pezsp_runtime::traits::TryConvert;
+
+/// Parameters that have the same names across all bridges.
+#[derive(Debug, PartialEq, Parser)]
+pub struct HeadersAndMessagesSharedParams {
+	/// Hex-encoded lane identifiers that should be served by the complex relay.
+	#[arg(long)]
+	pub lane: Vec<HexLaneId>,
+	/// If passed, only mandatory headers (headers that are changing the GRANDPA authorities set)
+	/// are relayed.
+	#[arg(long)]
+	pub only_mandatory_headers: bool,
+	/// If passed, only free headers (mandatory and every Nth header, if configured in runtime)
+	/// are relayed. Overrides `only_mandatory_headers`.
+	#[arg(long)]
+	pub only_free_headers: bool,
+	#[command(flatten)]
+	/// Prometheus metrics params.
+	pub prometheus_params: PrometheusParams,
+}
+
+impl HeadersAndMessagesSharedParams {
+	fn headers_to_relay(&self) -> HeadersToRelay {
+		match (self.only_mandatory_headers, self.only_free_headers) {
+			(_, true) => HeadersToRelay::Free,
+			(true, false) => HeadersToRelay::Mandatory,
+			_ => HeadersToRelay::All,
+		}
+	}
+}
+
+/// Bridge parameters, shared by all bridge types.
+pub struct Full2WayBridgeCommonParams<
+	Left: ChainWithTransactions + ChainWithRuntimeVersion,
+	Right: ChainWithTransactions + ChainWithRuntimeVersion,
+> {
+	/// Shared parameters.
+	pub shared: HeadersAndMessagesSharedParams,
+	/// Parameters of the left chain.
+	pub left: BridgeEndCommonParams<Left>,
+	/// Parameters of the right chain.
+	pub right: BridgeEndCommonParams<Right>,
+
+	/// Common metric parameters.
+	pub metrics_params: MetricsParams,
+}
+
+impl<
+		Left: ChainWithTransactions + ChainWithRuntimeVersion,
+		Right: ChainWithTransactions + ChainWithRuntimeVersion,
+	> Full2WayBridgeCommonParams<Left, Right>
+{
+	/// Creates new bridge parameters from its components.
+	pub fn new<L2R: MessagesCliBridge<Source = Left, Target = Right>>(
+		shared: HeadersAndMessagesSharedParams,
+		left: BridgeEndCommonParams<Left>,
+		right: BridgeEndCommonParams<Right>,
+	) -> anyhow::Result<Self> {
+		// Create metrics registry.
+		let metrics_params = shared.prometheus_params.clone().into_metrics_params()?;
+		let metrics_params = relay_utils::relay_metrics(metrics_params).into_params();
+
+		Ok(Self { shared, left, right, metrics_params })
+	}
+}
+
+/// Parameters that are associated with one side of the bridge.
+pub struct BridgeEndCommonParams<Chain: ChainWithTransactions + ChainWithRuntimeVersion> {
+	/// Chain client.
+	pub client: DefaultClient<Chain>,
+	/// Params used for sending transactions to the chain.
+	pub tx_params: TransactionParams<AccountKeyPairOf<Chain>>,
+	/// Accounts, which balances are exposed as metrics by the relay process.
+	pub accounts: Vec<TaggedAccount<AccountIdOf<Chain>>>,
+}
+
+/// All data of the bidirectional complex relay.
+pub struct FullBridge<
+	'a,
+	Source: ChainWithTransactions + ChainWithRuntimeVersion,
+	Target: ChainWithTransactions + ChainWithRuntimeVersion,
+	Bridge: MessagesCliBridge<Source = Source, Target = Target>,
+> {
+	source: &'a mut BridgeEndCommonParams<Source>,
+	target: &'a mut BridgeEndCommonParams<Target>,
+	metrics_params: &'a MetricsParams,
+	_phantom_data: PhantomData<Bridge>,
+}
+
+impl<
+		'a,
+		Source: ChainWithTransactions + ChainWithRuntimeVersion,
+		Target: ChainWithTransactions + ChainWithRuntimeVersion,
+		Bridge: MessagesCliBridge<Source = Source, Target = Target>,
+	> FullBridge<'a, Source, Target, Bridge>
+where
+	AccountIdOf<Source>: From<<AccountKeyPairOf<Source> as Pair>::Public>,
+	AccountIdOf<Target>: From<<AccountKeyPairOf<Target> as Pair>::Public>,
+	BalanceOf<Source>: TryFrom<BalanceOf<Target>> + Into<u128>,
+{
+	/// Construct complex relay given it components.
+	fn new(
+		source: &'a mut BridgeEndCommonParams<Source>,
+		target: &'a mut BridgeEndCommonParams<Target>,
+		metrics_params: &'a MetricsParams,
+	) -> Self {
+		Self { source, target, metrics_params, _phantom_data: Default::default() }
+	}
+
+	/// Returns message relay parameters.
+	fn pez_messages_relay_params(
+		&self,
+		source_to_target_headers_relay: Arc<dyn OnDemandRelay<Source, Target>>,
+		target_to_source_headers_relay: Arc<dyn OnDemandRelay<Target, Source>>,
+		lane_id: MessagesLaneIdOf<Bridge>,
+		maybe_limits: Option<MessagesRelayLimits>,
+	) -> MessagesRelayParams<Bridge::MessagesLane, DefaultClient<Source>, DefaultClient<Target>> {
+		MessagesRelayParams {
+			source_client: self.source.client.clone(),
+			source_transaction_params: self.source.tx_params.clone(),
+			target_client: self.target.client.clone(),
+			target_transaction_params: self.target.tx_params.clone(),
+			source_to_target_headers_relay: Some(source_to_target_headers_relay),
+			target_to_source_headers_relay: Some(target_to_source_headers_relay),
+			lane_id,
+			limits: maybe_limits,
+			metrics_params: self.metrics_params.clone().disable(),
+		}
+	}
+}
+
+/// Base portion of the bidirectional complex relay.
+///
+/// This main purpose of extracting this trait is that in different relays the implementation
+/// of `start_on_demand_headers_relayers` method will be different. But the number of
+/// implementations is limited to relay <> relay, teyrchain <> relay and teyrchain <> teyrchain.
+/// This trait allows us to reuse these implementations in different bridges.
+#[async_trait]
+pub trait Full2WayBridgeBase: Sized + Send + Sync {
+	/// The CLI params for the bridge.
+	type Params;
+	/// The left relay chain.
+	type Left: ChainWithTransactions + ChainWithRuntimeVersion;
+	/// The right destination chain (it can be a relay or a teyrchain).
+	type Right: ChainWithTransactions + ChainWithRuntimeVersion;
+
+	/// Reference to common relay parameters.
+	fn common(&self) -> &Full2WayBridgeCommonParams<Self::Left, Self::Right>;
+
+	/// Mutable reference to common relay parameters.
+	fn mut_common(&mut self) -> &mut Full2WayBridgeCommonParams<Self::Left, Self::Right>;
+
+	/// Start on-demand headers relays.
+	async fn start_on_demand_headers_relayers(
+		&mut self,
+	) -> anyhow::Result<(
+		Arc<dyn OnDemandRelay<Self::Left, Self::Right>>,
+		Arc<dyn OnDemandRelay<Self::Right, Self::Left>>,
+	)>;
+}
+
+/// Bidirectional complex relay.
+#[async_trait]
+pub trait Full2WayBridge: Sized + Sync
+where
+	AccountIdOf<Self::Left>: From<<AccountKeyPairOf<Self::Left> as Pair>::Public>,
+	AccountIdOf<Self::Right>: From<<AccountKeyPairOf<Self::Right> as Pair>::Public>,
+	BalanceOf<Self::Left>: TryFrom<BalanceOf<Self::Right>> + Into<u128>,
+	BalanceOf<Self::Right>: TryFrom<BalanceOf<Self::Left>> + Into<u128>,
+{
+	/// Base portion of the bidirectional complex relay.
+	type Base: Full2WayBridgeBase<Left = Self::Left, Right = Self::Right>;
+
+	/// The left relay chain.
+	type Left: ChainWithTransactions
+		+ ChainWithBalances
+		+ ChainWithMessages
+		+ ChainWithRuntimeVersion;
+	/// The right relay chain.
+	type Right: ChainWithTransactions
+		+ ChainWithBalances
+		+ ChainWithMessages
+		+ ChainWithRuntimeVersion;
+
+	/// Left to Right bridge.
+	type L2R: MessagesCliBridge<Source = Self::Left, Target = Self::Right>;
+	/// Right to Left bridge
+	type R2L: MessagesCliBridge<Source = Self::Right, Target = Self::Left>;
+
+	/// Construct new bridge.
+	fn new(params: <Self::Base as Full2WayBridgeBase>::Params) -> anyhow::Result<Self>;
+
+	/// Reference to the base relay portion.
+	fn base(&self) -> &Self::Base;
+
+	/// Mutable reference to the base relay portion.
+	fn mut_base(&mut self) -> &mut Self::Base;
+
+	/// Creates and returns Left to Right complex relay.
+	fn left_to_right(&mut self) -> FullBridge<'_, Self::Left, Self::Right, Self::L2R> {
+		let common = self.mut_base().mut_common();
+		FullBridge::<_, _, Self::L2R>::new(
+			&mut common.left,
+			&mut common.right,
+			&common.metrics_params,
+		)
+	}
+
+	/// Creates and returns Right to Left complex relay.
+	fn right_to_left(&mut self) -> FullBridge<'_, Self::Right, Self::Left, Self::R2L> {
+		let common = self.mut_base().mut_common();
+		FullBridge::<_, _, Self::R2L>::new(
+			&mut common.right,
+			&mut common.left,
+			&common.metrics_params,
+		)
+	}
+
+	/// Start complex relay.
+	async fn run(&mut self) -> anyhow::Result<()> {
+		// Register standalone metrics.
+		{
+			let common = self.mut_base().mut_common();
+			common.left.accounts.push(TaggedAccount::Messages {
+				id: common.left.tx_params.signer.public().into(),
+				bridged_chain: Self::Right::NAME.to_string(),
+			});
+			common.right.accounts.push(TaggedAccount::Messages {
+				id: common.right.tx_params.signer.public().into(),
+				bridged_chain: Self::Left::NAME.to_string(),
+			});
+		}
+
+		// start on-demand header relays
+		let (left_to_right_on_demand_headers, right_to_left_on_demand_headers) =
+			self.mut_base().start_on_demand_headers_relayers().await?;
+
+		// add balance-related metrics
+		let lanes_l2r: Vec<MessagesLaneIdOf<Self::L2R>> = self
+			.base()
+			.common()
+			.shared
+			.lane
+			.iter()
+			.cloned()
+			.map(HexLaneId::try_convert)
+			.collect::<Result<Vec<_>, HexLaneId>>()
+			.map_err(|e| {
+				anyhow::format_err!("Conversion failed for L2R lanes with error: {:?}!", e)
+			})?;
+		let lanes_r2l: Vec<MessagesLaneIdOf<Self::R2L>> = self
+			.base()
+			.common()
+			.shared
+			.lane
+			.iter()
+			.cloned()
+			.map(HexLaneId::try_convert)
+			.collect::<Result<Vec<_>, HexLaneId>>()
+			.map_err(|e| {
+				anyhow::format_err!("Conversion failed for R2L lanes with error: {:?}!", e)
+			})?;
+		{
+			let common = self.mut_base().mut_common();
+			crate::messages::metrics::add_relay_balances_metrics::<_>(
+				common.left.client.clone(),
+				&common.metrics_params,
+				&common.left.accounts,
+			)
+			.await?;
+			crate::messages::metrics::add_relay_balances_metrics::<_>(
+				common.right.client.clone(),
+				&common.metrics_params,
+				&common.right.accounts,
+			)
+			.await?;
+		}
+
+		// Need 2x capacity since we consider both directions for each lane
+		let mut message_relays =
+			Vec::with_capacity(lanes_l2r.len().saturating_add(lanes_r2l.len()));
+		for lane in lanes_l2r {
+			let left_to_right_messages =
+				crate::messages::run::<<Self::L2R as MessagesCliBridge>::MessagesLane, _, _>(
+					self.left_to_right().pez_messages_relay_params(
+						left_to_right_on_demand_headers.clone(),
+						right_to_left_on_demand_headers.clone(),
+						lane,
+						Self::L2R::maybe_messages_limits(),
+					),
+				)
+				.map_err(|e| anyhow::format_err!("{}", e))
+				.boxed();
+			message_relays.push(left_to_right_messages);
+		}
+		for lane in lanes_r2l {
+			let right_to_left_messages =
+				crate::messages::run::<<Self::R2L as MessagesCliBridge>::MessagesLane, _, _>(
+					self.right_to_left().pez_messages_relay_params(
+						right_to_left_on_demand_headers.clone(),
+						left_to_right_on_demand_headers.clone(),
+						lane,
+						Self::R2L::maybe_messages_limits(),
+					),
+				)
+				.map_err(|e| anyhow::format_err!("{}", e))
+				.boxed();
+			message_relays.push(right_to_left_messages);
+		}
+
+		relay_utils::relay_metrics(self.base().common().metrics_params.clone())
+			.expose()
+			.await
+			.map_err(|e| anyhow::format_err!("{}", e))?;
+
+		futures::future::select_all(message_relays).await.0
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::{cli::chain_schema::RuntimeVersionType, declare_chain_cli_schema};
+
+	use relay_bizinikiwi_client::{ChainRuntimeVersion, SimpleRuntimeVersion, Teyrchain};
+
+	#[test]
+	// We need `#[allow(dead_code)]` because some of the methods generated by the macros
+	// are not used.
+	#[allow(dead_code)]
+	fn should_parse_teyrchain_to_teyrchain_options() {
+		// Chains.
+		declare_chain_cli_schema!(Kusama, kusama);
+		declare_chain_cli_schema!(BridgeHubKusama, bridge_hub_kusama);
+		declare_chain_cli_schema!(Pezkuwi, pezkuwi);
+		declare_chain_cli_schema!(BridgeHubPezkuwi, bridge_hub_pezkuwi);
+		// Means to override signers of different layer transactions.
+		declare_chain_cli_schema!(
+			KusamaHeadersToBridgeHubPezkuwi,
+			kusama_headers_to_bridge_hub_pezkuwi
+		);
+		declare_chain_cli_schema!(
+			KusamaTeyrchainsToBridgeHubPezkuwi,
+			kusama_teyrchains_to_bridge_hub_pezkuwi
+		);
+		declare_chain_cli_schema!(
+			PezkuwiHeadersToBridgeHubKusama,
+			pezkuwi_headers_to_bridge_hub_kusama
+		);
+		declare_chain_cli_schema!(
+			PezkuwiTeyrchainsToBridgeHubKusama,
+			pezkuwi_teyrchains_to_bridge_hub_kusama
+		);
+		// Bridges.
+		declare_teyrchain_to_teyrchain_bridge_schema!(
+			BridgeHubKusama,
+			Kusama,
+			BridgeHubPezkuwi,
+			Pezkuwi
+		);
+
+		let res = BridgeHubKusamaBridgeHubPezkuwiHeadersAndMessages::parse_from(vec![
+			"bridge-hub-kusama-bridge-hub-pezkuwi-headers-and-messages",
+			"--bridge-hub-kusama-uri",
+			"ws://bridge-hub-zagros-collator1:9944",
+			"--bridge-hub-kusama-signer",
+			"//Iden",
+			"--bridge-hub-kusama-transactions-mortality",
+			"64",
+			"--kusama-uri",
+			"ws://zagros-alice:9944",
+			"--bridge-hub-pezkuwi-uri",
+			"ws://bridge-hub-pezkuwichain-collator1:9944",
+			"--bridge-hub-pezkuwi-signer",
+			"//George",
+			"--bridge-hub-pezkuwi-transactions-mortality",
+			"64",
+			"--pezkuwi-uri",
+			"ws://pezkuwichain-alice:9944",
+			"--lane",
+			"0000000000000000000000000000000000000000000000000000000000000000",
+			"--prometheus-host",
+			"0.0.0.0",
+		]);
+
+		// then
+		assert_eq!(
+			res,
+			BridgeHubKusamaBridgeHubPezkuwiHeadersAndMessages {
+				shared: HeadersAndMessagesSharedParams {
+					lane: vec![HexLaneId(vec![0x00u8; 32])],
+					only_mandatory_headers: false,
+					only_free_headers: false,
+					prometheus_params: PrometheusParams {
+						no_prometheus: false,
+						prometheus_host: "0.0.0.0".into(),
+						prometheus_port: 9616,
+					},
+				},
+				left: BridgeHubKusamaConnectionParams {
+					bridge_hub_kusama_uri: "ws://bridge-hub-zagros-collator1:9944".into(),
+					bridge_hub_kusama_runtime_version: BridgeHubKusamaRuntimeVersionParams {
+						bridge_hub_kusama_version_mode: RuntimeVersionType::Bundle,
+						bridge_hub_kusama_spec_version: None,
+						bridge_hub_kusama_transaction_version: None,
+					},
+				},
+				left_sign: BridgeHubKusamaSigningParams {
+					bridge_hub_kusama_signer: Some("//Iden".into()),
+					bridge_hub_kusama_signer_password: None,
+					bridge_hub_kusama_signer_file: None,
+					bridge_hub_kusama_signer_password_file: None,
+					bridge_hub_kusama_transactions_mortality: Some(64),
+				},
+				left_relay: KusamaConnectionParams {
+					kusama_uri: "ws://zagros-alice:9944".into(),
+					kusama_runtime_version: KusamaRuntimeVersionParams {
+						kusama_version_mode: RuntimeVersionType::Bundle,
+						kusama_spec_version: None,
+						kusama_transaction_version: None,
+					},
+				},
+				right: BridgeHubPezkuwiConnectionParams {
+					bridge_hub_pezkuwi_uri: "ws://bridge-hub-pezkuwichain-collator1:9944".into(),
+					bridge_hub_pezkuwi_runtime_version: BridgeHubPezkuwiRuntimeVersionParams {
+						bridge_hub_pezkuwi_version_mode: RuntimeVersionType::Bundle,
+						bridge_hub_pezkuwi_spec_version: None,
+						bridge_hub_pezkuwi_transaction_version: None,
+					},
+				},
+				right_sign: BridgeHubPezkuwiSigningParams {
+					bridge_hub_pezkuwi_signer: Some("//George".into()),
+					bridge_hub_pezkuwi_signer_password: None,
+					bridge_hub_pezkuwi_signer_file: None,
+					bridge_hub_pezkuwi_signer_password_file: None,
+					bridge_hub_pezkuwi_transactions_mortality: Some(64),
+				},
+				right_relay: PezkuwiConnectionParams {
+					pezkuwi_uri: "ws://pezkuwichain-alice:9944".into(),
+					pezkuwi_runtime_version: PezkuwiRuntimeVersionParams {
+						pezkuwi_version_mode: RuntimeVersionType::Bundle,
+						pezkuwi_spec_version: None,
+						pezkuwi_transaction_version: None,
+					},
+				},
+			}
+		);
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/cli/relay_headers_and_messages/relay_to_relay.rs

@@ -0,0 +1,169 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+// we don't have any relay/standalone <> relay/standalone chain bridges, but we may need it in a
+// future
+#![allow(unused_macros)]
+
+//! Relay chain to Relay chain relayer CLI primitives.
+
+use async_trait::async_trait;
+use std::sync::Arc;
+
+use crate::{
+	cli::{
+		bridge::{CliBridgeBase, MessagesCliBridge, RelayToRelayHeadersCliBridge},
+		relay_headers_and_messages::{Full2WayBridgeBase, Full2WayBridgeCommonParams},
+	},
+	finality::BizinikiwiFinalitySyncPipeline,
+	on_demand::{headers::OnDemandHeadersRelay, OnDemandRelay},
+};
+use relay_bizinikiwi_client::{
+	AccountIdOf, AccountKeyPairOf, ChainWithRuntimeVersion, ChainWithTransactions, Client,
+};
+use pezsp_core::Pair;
+
+/// A base relay between two standalone (relay) chains.
+///
+/// Such relay starts 2 messages relay and 2 on-demand header relays.
+pub struct RelayToRelayBridge<
+	L2R: MessagesCliBridge + RelayToRelayHeadersCliBridge,
+	R2L: MessagesCliBridge + RelayToRelayHeadersCliBridge,
+> {
+	/// Parameters that are shared by all bridge types.
+	pub common:
+		Full2WayBridgeCommonParams<<R2L as CliBridgeBase>::Target, <L2R as CliBridgeBase>::Target>,
+}
+
+/// Create set of configuration objects specific to relay-to-relay relayer.
+macro_rules! declare_relay_to_relay_bridge_schema {
+	($left_chain:ident, $right_chain:ident) => {
+		pezbp_runtime::paste::item! {
+			#[doc = $left_chain " and " $right_chain " headers+messages relay params."]
+			#[derive(Debug, PartialEq, Parser)]
+			pub struct [<$left_chain $right_chain HeadersAndMessages>] {
+				#[command(flatten)]
+				shared: HeadersAndMessagesSharedParams,
+
+				#[command(flatten)]
+				left: [<$left_chain ConnectionParams>],
+				// default signer, which is always used to sign messages relay transactions on the left chain
+				#[command(flatten)]
+				left_sign: [<$left_chain SigningParams>],
+
+				#[command(flatten)]
+				right: [<$right_chain ConnectionParams>],
+				#[command(flatten)]
+				// default signer, which is always used to sign messages relay transactions on the right chain
+				right_sign: [<$right_chain SigningParams>],
+			}
+
+			impl [<$left_chain $right_chain HeadersAndMessages>] {
+				async fn into_bridge<
+					Left: ChainWithTransactions + CliChain,
+					Right: ChainWithTransactions + CliChain,
+					L2R: CliBridgeBase<Source = Left, Target = Right> + MessagesCliBridge + RelayToRelayHeadersCliBridge,
+					R2L: CliBridgeBase<Source = Right, Target = Left> + MessagesCliBridge + RelayToRelayHeadersCliBridge,
+				>(
+					self,
+				) -> anyhow::Result<RelayToRelayBridge<L2R, R2L>> {
+					Ok(RelayToRelayBridge {
+						common: Full2WayBridgeCommonParams::new::<L2R>(
+							self.shared,
+							BridgeEndCommonParams {
+								client: self.left.into_client::<Left>().await?,
+								tx_params: self.left_sign.transaction_params::<Left>()?,
+								accounts: vec![],
+							},
+							BridgeEndCommonParams {
+								client: self.right.into_client::<Right>().await?,
+								tx_params: self.right_sign.transaction_params::<Right>()?,
+								accounts: vec![],
+							},
+						)?,
+						right_to_left_transaction_params: self.left_sign.transaction_params::<Left>(),
+						left_to_right_transaction_params: self.right_sign.transaction_params::<Right>(),
+					})
+				}
+			}
+		}
+	};
+}
+
+#[async_trait]
+impl<
+		Left: ChainWithTransactions + ChainWithRuntimeVersion,
+		Right: ChainWithTransactions + ChainWithRuntimeVersion,
+		L2R: CliBridgeBase<Source = Left, Target = Right>
+			+ MessagesCliBridge
+			+ RelayToRelayHeadersCliBridge,
+		R2L: CliBridgeBase<Source = Right, Target = Left>
+			+ MessagesCliBridge
+			+ RelayToRelayHeadersCliBridge,
+	> Full2WayBridgeBase for RelayToRelayBridge<L2R, R2L>
+where
+	AccountIdOf<Left>: From<<AccountKeyPairOf<Left> as Pair>::Public>,
+	AccountIdOf<Right>: From<<AccountKeyPairOf<Right> as Pair>::Public>,
+{
+	type Params = RelayToRelayBridge<L2R, R2L>;
+	type Left = Left;
+	type Right = Right;
+
+	fn common(&self) -> &Full2WayBridgeCommonParams<Left, Right> {
+		&self.common
+	}
+
+	fn mut_common(&mut self) -> &mut Full2WayBridgeCommonParams<Self::Left, Self::Right> {
+		&mut self.common
+	}
+
+	async fn start_on_demand_headers_relayers(
+		&mut self,
+	) -> anyhow::Result<(
+		Arc<dyn OnDemandRelay<Self::Left, Self::Right>>,
+		Arc<dyn OnDemandRelay<Self::Right, Self::Left>>,
+	)> {
+		<L2R as RelayToRelayHeadersCliBridge>::Finality::start_relay_guards(
+			&self.common.right.client,
+			self.common.right.client.can_start_version_guard(),
+		)
+		.await?;
+		<R2L as RelayToRelayHeadersCliBridge>::Finality::start_relay_guards(
+			&self.common.left.client,
+			self.common.left.client.can_start_version_guard(),
+		)
+		.await?;
+
+		let left_to_right_on_demand_headers =
+			OnDemandHeadersRelay::<<L2R as RelayToRelayHeadersCliBridge>::Finality, _, _>::new(
+				self.common.left.client.clone(),
+				self.common.right.client.clone(),
+				self.common.right.tx_params.clone(),
+				self.common.shared.headers_to_relay(),
+				None,
+			);
+		let right_to_left_on_demand_headers =
+			OnDemandHeadersRelay::<<R2L as RelayToRelayHeadersCliBridge>::Finality, _, _>::new(
+				self.common.right.client.clone(),
+				self.common.left.client.clone(),
+				self.common.left.tx_params.clone(),
+				self.common.shared.headers_to_relay(),
+				None,
+			);
+
+		Ok((Arc::new(left_to_right_on_demand_headers), Arc::new(right_to_left_on_demand_headers)))
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/cli/relay_headers_and_messages/relay_to_teyrchain.rs

@@ -0,0 +1,205 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Relay chain to teyrchain relayer CLI primitives.
+
+use async_trait::async_trait;
+use std::sync::Arc;
+
+use crate::{
+	cli::{
+		bridge::{
+			CliBridgeBase, MessagesCliBridge, RelayToRelayHeadersCliBridge,
+			TeyrchainToRelayHeadersCliBridge,
+		},
+		relay_headers_and_messages::{Full2WayBridgeBase, Full2WayBridgeCommonParams},
+		DefaultClient,
+	},
+	finality::BizinikiwiFinalitySyncPipeline,
+	on_demand::{
+		headers::OnDemandHeadersRelay, teyrchains::OnDemandTeyrchainsRelay, OnDemandRelay,
+	},
+};
+use bp_pezkuwi_core::teyrchains::ParaHash;
+use bp_teyrchains::{RelayBlockHash, RelayBlockHasher, RelayBlockNumber};
+use relay_bizinikiwi_client::{
+	AccountIdOf, AccountKeyPairOf, Chain, ChainWithRuntimeVersion, ChainWithTransactions, Client,
+	Teyrchain,
+};
+use pezsp_core::Pair;
+
+/// A base relay between standalone (relay) chain and a teyrchain from another consensus system.
+///
+/// Such relay starts 2 messages relay. It also starts 2 on-demand header relays and 1 on-demand
+/// teyrchain heads relay.
+pub struct RelayToTeyrchainBridge<
+	L2R: MessagesCliBridge + RelayToRelayHeadersCliBridge,
+	R2L: MessagesCliBridge + TeyrchainToRelayHeadersCliBridge,
+> where
+	<R2L as CliBridgeBase>::Source: Teyrchain,
+{
+	/// Parameters that are shared by all bridge types.
+	pub common:
+		Full2WayBridgeCommonParams<<R2L as CliBridgeBase>::Target, <L2R as CliBridgeBase>::Target>,
+	/// Client of the right relay chain.
+	pub right_relay: DefaultClient<<R2L as TeyrchainToRelayHeadersCliBridge>::SourceRelay>,
+}
+
+/// Create set of configuration objects specific to relay-to-teyrchain relayer.
+#[macro_export]
+macro_rules! declare_relay_to_teyrchain_bridge_schema {
+	// chain, teyrchain, relay-chain-of-teyrchain
+	($left_chain:ident, $right_teyrchain:ident, $right_chain:ident) => {
+		pezbp_runtime::paste::item! {
+			#[doc = $left_chain ", " $right_teyrchain " and " $right_chain " headers+teyrchains+messages relay params."]
+			#[derive(Debug, PartialEq, Parser)]
+			pub struct [<$left_chain $right_teyrchain HeadersAndMessages>] {
+				// shared parameters
+				#[command(flatten)]
+				shared: HeadersAndMessagesSharedParams,
+
+				#[command(flatten)]
+				left: [<$left_chain ConnectionParams>],
+				// default signer, which is always used to sign messages relay transactions on the left chain
+				#[command(flatten)]
+				left_sign: [<$left_chain SigningParams>],
+
+				#[command(flatten)]
+				right: [<$right_teyrchain ConnectionParams>],
+				// default signer, which is always used to sign messages relay transactions on the right chain
+				#[command(flatten)]
+				right_sign: [<$right_teyrchain SigningParams>],
+
+				#[command(flatten)]
+				right_relay: [<$right_chain ConnectionParams>],
+			}
+
+			impl [<$left_chain $right_teyrchain HeadersAndMessages>] {
+				async fn into_bridge<
+					Left: ChainWithTransactions + ChainWithRuntimeVersion,
+					Right: ChainWithTransactions + ChainWithRuntimeVersion + Teyrchain,
+					RightRelay: ChainWithRuntimeVersion,
+					L2R: CliBridgeBase<Source = Left, Target = Right> + MessagesCliBridge + RelayToRelayHeadersCliBridge,
+					R2L: CliBridgeBase<Source = Right, Target = Left>
+						+ MessagesCliBridge
+						+ TeyrchainToRelayHeadersCliBridge<SourceRelay = RightRelay>,
+				>(
+					self,
+				) -> anyhow::Result<RelayToTeyrchainBridge<L2R, R2L>> {
+					Ok(RelayToTeyrchainBridge {
+						common: Full2WayBridgeCommonParams::new::<L2R>(
+							self.shared,
+							BridgeEndCommonParams {
+								client: self.left.into_client::<Left>().await?,
+								tx_params: self.left_sign.transaction_params::<Left>()?,
+								accounts: vec![],
+							},
+							BridgeEndCommonParams {
+								client: self.right.into_client::<Right>().await?,
+								tx_params: self.right_sign.transaction_params::<Right>()?,
+								accounts: vec![],
+							},
+						)?,
+						right_relay: self.right_relay.into_client::<RightRelay>().await?,
+					})
+				}
+			}
+		}
+	};
+}
+
+#[async_trait]
+impl<
+		Left: ChainWithTransactions + ChainWithRuntimeVersion,
+		Right: Chain<Hash = ParaHash> + ChainWithTransactions + ChainWithRuntimeVersion + Teyrchain,
+		RightRelay: Chain<BlockNumber = RelayBlockNumber, Hash = RelayBlockHash, Hasher = RelayBlockHasher>
+			+ ChainWithRuntimeVersion,
+		L2R: CliBridgeBase<Source = Left, Target = Right>
+			+ MessagesCliBridge
+			+ RelayToRelayHeadersCliBridge,
+		R2L: CliBridgeBase<Source = Right, Target = Left>
+			+ MessagesCliBridge
+			+ TeyrchainToRelayHeadersCliBridge<SourceRelay = RightRelay>,
+	> Full2WayBridgeBase for RelayToTeyrchainBridge<L2R, R2L>
+where
+	AccountIdOf<Left>: From<<AccountKeyPairOf<Left> as Pair>::Public>,
+	AccountIdOf<Right>: From<<AccountKeyPairOf<Right> as Pair>::Public>,
+{
+	type Params = RelayToTeyrchainBridge<L2R, R2L>;
+	type Left = Left;
+	type Right = Right;
+
+	fn common(&self) -> &Full2WayBridgeCommonParams<Left, Right> {
+		&self.common
+	}
+
+	fn mut_common(&mut self) -> &mut Full2WayBridgeCommonParams<Self::Left, Self::Right> {
+		&mut self.common
+	}
+
+	async fn start_on_demand_headers_relayers(
+		&mut self,
+	) -> anyhow::Result<(
+		Arc<dyn OnDemandRelay<Self::Left, Self::Right>>,
+		Arc<dyn OnDemandRelay<Self::Right, Self::Left>>,
+	)> {
+		<L2R as RelayToRelayHeadersCliBridge>::Finality::start_relay_guards(
+			&self.common.right.client,
+			self.common.right.client.can_start_version_guard(),
+		)
+		.await?;
+		<R2L as TeyrchainToRelayHeadersCliBridge>::RelayFinality::start_relay_guards(
+			&self.common.left.client,
+			self.common.left.client.can_start_version_guard(),
+		)
+		.await?;
+
+		let left_to_right_on_demand_headers =
+			OnDemandHeadersRelay::<<L2R as RelayToRelayHeadersCliBridge>::Finality, _, _>::new(
+				self.common.left.client.clone(),
+				self.common.right.client.clone(),
+				self.common.right.tx_params.clone(),
+				self.common.shared.headers_to_relay(),
+				None,
+			);
+		let right_relay_to_left_on_demand_headers = OnDemandHeadersRelay::<
+			<R2L as TeyrchainToRelayHeadersCliBridge>::RelayFinality,
+			_,
+			_,
+		>::new(
+			self.right_relay.clone(),
+			self.common.left.client.clone(),
+			self.common.left.tx_params.clone(),
+			self.common.shared.headers_to_relay(),
+			Some(self.common.metrics_params.clone()),
+		);
+		let right_to_left_on_demand_teyrchains = OnDemandTeyrchainsRelay::<
+			<R2L as TeyrchainToRelayHeadersCliBridge>::TeyrchainFinality,
+			_,
+			_,
+		>::new(
+			self.right_relay.clone(),
+			self.common.left.client.clone(),
+			self.common.left.tx_params.clone(),
+			Arc::new(right_relay_to_left_on_demand_headers),
+		);
+
+		Ok((
+			Arc::new(left_to_right_on_demand_headers),
+			Arc::new(right_to_left_on_demand_teyrchains),
+		))
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/cli/relay_headers_and_messages/teyrchain_to_teyrchain.rs

@@ -0,0 +1,228 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Teyrchain to teyrchain relayer CLI primitives.
+
+use async_trait::async_trait;
+use std::sync::Arc;
+
+use crate::{
+	cli::{
+		bridge::{CliBridgeBase, MessagesCliBridge, TeyrchainToRelayHeadersCliBridge},
+		relay_headers_and_messages::{Full2WayBridgeBase, Full2WayBridgeCommonParams},
+		DefaultClient,
+	},
+	finality::BizinikiwiFinalitySyncPipeline,
+	on_demand::{
+		headers::OnDemandHeadersRelay, teyrchains::OnDemandTeyrchainsRelay, OnDemandRelay,
+	},
+};
+use bp_pezkuwi_core::teyrchains::ParaHash;
+use bp_teyrchains::{RelayBlockHash, RelayBlockHasher, RelayBlockNumber};
+use relay_bizinikiwi_client::{
+	AccountIdOf, AccountKeyPairOf, Chain, ChainWithRuntimeVersion, ChainWithTransactions, Client,
+	Teyrchain,
+};
+use pezsp_core::Pair;
+
+/// A base relay between two teyrchain from different consensus systems.
+///
+/// Such relay starts 2 messages relay. It also starts 2 on-demand header relays and 2 on-demand
+/// teyrchain heads relay.
+pub struct TeyrchainToTeyrchainBridge<
+	L2R: MessagesCliBridge + TeyrchainToRelayHeadersCliBridge,
+	R2L: MessagesCliBridge + TeyrchainToRelayHeadersCliBridge,
+> where
+	<L2R as CliBridgeBase>::Source: Teyrchain,
+	<R2L as CliBridgeBase>::Source: Teyrchain,
+{
+	/// Parameters that are shared by all bridge types.
+	pub common:
+		Full2WayBridgeCommonParams<<R2L as CliBridgeBase>::Target, <L2R as CliBridgeBase>::Target>,
+	/// Client of the left relay chain.
+	pub left_relay: DefaultClient<<L2R as TeyrchainToRelayHeadersCliBridge>::SourceRelay>,
+	/// Client of the right relay chain.
+	pub right_relay: DefaultClient<<R2L as TeyrchainToRelayHeadersCliBridge>::SourceRelay>,
+}
+
+/// Create set of configuration objects specific to teyrchain-to-teyrchain relayer.
+#[macro_export]
+macro_rules! declare_teyrchain_to_teyrchain_bridge_schema {
+	// left-teyrchain, relay-chain-of-left-teyrchain, right-teyrchain, relay-chain-of-right-teyrchain
+	($left_teyrchain:ident, $left_chain:ident, $right_teyrchain:ident, $right_chain:ident) => {
+		pezbp_runtime::paste::item! {
+			#[doc = $left_teyrchain ", " $left_chain ", " $right_teyrchain " and " $right_chain " headers+teyrchains+messages relay params."]
+			#[derive(Debug, PartialEq, Parser)]
+			pub struct [<$left_teyrchain $right_teyrchain HeadersAndMessages>] {
+				// shared parameters
+				#[command(flatten)]
+				shared: HeadersAndMessagesSharedParams,
+
+				#[command(flatten)]
+				left: [<$left_teyrchain ConnectionParams>],
+				// default signer, which is always used to sign messages relay transactions on the left chain
+				#[command(flatten)]
+				left_sign: [<$left_teyrchain SigningParams>],
+
+				#[command(flatten)]
+				left_relay: [<$left_chain ConnectionParams>],
+
+				#[command(flatten)]
+				right: [<$right_teyrchain ConnectionParams>],
+				// default signer, which is always used to sign messages relay transactions on the right chain
+				#[command(flatten)]
+				right_sign: [<$right_teyrchain SigningParams>],
+
+				#[command(flatten)]
+				right_relay: [<$right_chain ConnectionParams>],
+			}
+
+			impl [<$left_teyrchain $right_teyrchain HeadersAndMessages>] {
+				async fn into_bridge<
+					Left: ChainWithTransactions + ChainWithRuntimeVersion + Teyrchain,
+					LeftRelay: ChainWithRuntimeVersion,
+					Right: ChainWithTransactions + ChainWithRuntimeVersion + Teyrchain,
+					RightRelay: ChainWithRuntimeVersion,
+					L2R: $crate::cli::bridge::CliBridgeBase<Source = Left, Target = Right>
+						+ MessagesCliBridge
+						+ $crate::cli::bridge::TeyrchainToRelayHeadersCliBridge<SourceRelay = LeftRelay>,
+					R2L: $crate::cli::bridge::CliBridgeBase<Source = Right, Target = Left>
+						+ MessagesCliBridge
+						+ $crate::cli::bridge::TeyrchainToRelayHeadersCliBridge<SourceRelay = RightRelay>,
+				>(
+					self,
+				) -> anyhow::Result<$crate::cli::relay_headers_and_messages::teyrchain_to_teyrchain::TeyrchainToTeyrchainBridge<L2R, R2L>> {
+					Ok($crate::cli::relay_headers_and_messages::teyrchain_to_teyrchain::TeyrchainToTeyrchainBridge {
+						common: Full2WayBridgeCommonParams::new::<L2R>(
+							self.shared,
+							BridgeEndCommonParams {
+								client: self.left.into_client::<Left>().await?,
+								tx_params: self.left_sign.transaction_params::<Left>()?,
+								accounts: vec![],
+							},
+							BridgeEndCommonParams {
+								client: self.right.into_client::<Right>().await?,
+								tx_params: self.right_sign.transaction_params::<Right>()?,
+								accounts: vec![],
+							},
+						)?,
+						left_relay: self.left_relay.into_client::<LeftRelay>().await?,
+						right_relay: self.right_relay.into_client::<RightRelay>().await?,
+					})
+				}
+			}
+		}
+	};
+}
+
+#[async_trait]
+impl<
+		Left: Chain<Hash = ParaHash> + ChainWithTransactions + ChainWithRuntimeVersion + Teyrchain,
+		Right: Chain<Hash = ParaHash> + ChainWithTransactions + ChainWithRuntimeVersion + Teyrchain,
+		LeftRelay: Chain<BlockNumber = RelayBlockNumber, Hash = RelayBlockHash, Hasher = RelayBlockHasher>
+			+ ChainWithRuntimeVersion,
+		RightRelay: Chain<BlockNumber = RelayBlockNumber, Hash = RelayBlockHash, Hasher = RelayBlockHasher>
+			+ ChainWithRuntimeVersion,
+		L2R: CliBridgeBase<Source = Left, Target = Right>
+			+ MessagesCliBridge
+			+ TeyrchainToRelayHeadersCliBridge<SourceRelay = LeftRelay>,
+		R2L: CliBridgeBase<Source = Right, Target = Left>
+			+ MessagesCliBridge
+			+ TeyrchainToRelayHeadersCliBridge<SourceRelay = RightRelay>,
+	> Full2WayBridgeBase for TeyrchainToTeyrchainBridge<L2R, R2L>
+where
+	AccountIdOf<Left>: From<<AccountKeyPairOf<Left> as Pair>::Public>,
+	AccountIdOf<Right>: From<<AccountKeyPairOf<Right> as Pair>::Public>,
+{
+	type Params = TeyrchainToTeyrchainBridge<L2R, R2L>;
+	type Left = Left;
+	type Right = Right;
+
+	fn common(&self) -> &Full2WayBridgeCommonParams<Left, Right> {
+		&self.common
+	}
+
+	fn mut_common(&mut self) -> &mut Full2WayBridgeCommonParams<Self::Left, Self::Right> {
+		&mut self.common
+	}
+
+	async fn start_on_demand_headers_relayers(
+		&mut self,
+	) -> anyhow::Result<(
+		Arc<dyn OnDemandRelay<Self::Left, Self::Right>>,
+		Arc<dyn OnDemandRelay<Self::Right, Self::Left>>,
+	)> {
+		<L2R as TeyrchainToRelayHeadersCliBridge>::RelayFinality::start_relay_guards(
+			&self.common.right.client,
+			self.common.right.client.can_start_version_guard(),
+		)
+		.await?;
+		<R2L as TeyrchainToRelayHeadersCliBridge>::RelayFinality::start_relay_guards(
+			&self.common.left.client,
+			self.common.left.client.can_start_version_guard(),
+		)
+		.await?;
+
+		let left_relay_to_right_on_demand_headers = OnDemandHeadersRelay::<
+			<L2R as TeyrchainToRelayHeadersCliBridge>::RelayFinality,
+			_,
+			_,
+		>::new(
+			self.left_relay.clone(),
+			self.common.right.client.clone(),
+			self.common.right.tx_params.clone(),
+			self.common.shared.headers_to_relay(),
+			Some(self.common.metrics_params.clone()),
+		);
+		let right_relay_to_left_on_demand_headers = OnDemandHeadersRelay::<
+			<R2L as TeyrchainToRelayHeadersCliBridge>::RelayFinality,
+			_,
+			_,
+		>::new(
+			self.right_relay.clone(),
+			self.common.left.client.clone(),
+			self.common.left.tx_params.clone(),
+			self.common.shared.headers_to_relay(),
+			Some(self.common.metrics_params.clone()),
+		);
+
+		let left_to_right_on_demand_teyrchains = OnDemandTeyrchainsRelay::<
+			<L2R as TeyrchainToRelayHeadersCliBridge>::TeyrchainFinality,
+			_,
+			_,
+		>::new(
+			self.left_relay.clone(),
+			self.common.right.client.clone(),
+			self.common.right.tx_params.clone(),
+			Arc::new(left_relay_to_right_on_demand_headers),
+		);
+		let right_to_left_on_demand_teyrchains = OnDemandTeyrchainsRelay::<
+			<R2L as TeyrchainToRelayHeadersCliBridge>::TeyrchainFinality,
+			_,
+			_,
+		>::new(
+			self.right_relay.clone(),
+			self.common.left.client.clone(),
+			self.common.left.tx_params.clone(),
+			Arc::new(right_relay_to_left_on_demand_headers),
+		);
+
+		Ok((
+			Arc::new(left_to_right_on_demand_teyrchains),
+			Arc::new(right_to_left_on_demand_teyrchains),
+		))
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/cli/relay_messages.rs

@@ -0,0 +1,237 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Primitives for exposing the messages relaying functionality in the CLI.
+
+use crate::{
+	cli::{bridge::*, chain_schema::*, HexLaneId, PrometheusParams},
+	messages::MessagesRelayParams,
+	TransactionParams,
+};
+
+use async_trait::async_trait;
+use clap::Parser;
+use pezsp_core::Pair;
+
+use bp_messages::MessageNonce;
+use pezbp_runtime::HeaderIdProvider;
+use relay_bizinikiwi_client::{
+	AccountIdOf, AccountKeyPairOf, BalanceOf, Chain, ChainWithRuntimeVersion,
+	ChainWithTransactions, Client,
+};
+use relay_utils::UniqueSaturatedInto;
+use pezsp_runtime::traits::TryConvert;
+
+/// Messages relaying params.
+#[derive(Parser)]
+pub struct RelayMessagesParams {
+	/// Hex-encoded lane id that should be served by the relay.
+	#[arg(long)]
+	lane: HexLaneId,
+	#[command(flatten)]
+	source: SourceConnectionParams,
+	#[command(flatten)]
+	source_sign: SourceSigningParams,
+	#[command(flatten)]
+	target: TargetConnectionParams,
+	#[command(flatten)]
+	target_sign: TargetSigningParams,
+	#[command(flatten)]
+	prometheus_params: PrometheusParams,
+}
+
+/// Messages range relaying params.
+#[derive(Parser)]
+pub struct RelayMessagesRangeParams {
+	/// Number of the source chain header that we will use to prepare a messages proof.
+	/// This header must be previously proved to the target chain.
+	#[arg(long)]
+	at_source_block: u128,
+	/// Hex-encoded lane id that should be served by the relay.
+	#[arg(long)]
+	lane: HexLaneId,
+	/// Nonce (inclusive) of the first message to relay.
+	#[arg(long)]
+	messages_start: MessageNonce,
+	/// Nonce (inclusive) of the last message to relay.
+	#[arg(long)]
+	messages_end: MessageNonce,
+	/// Whether the outbound lane state proof should be included into transaction.
+	#[arg(long)]
+	outbound_state_proof_required: bool,
+	#[command(flatten)]
+	source: SourceConnectionParams,
+	#[command(flatten)]
+	source_sign: SourceSigningParams,
+	#[command(flatten)]
+	target: TargetConnectionParams,
+	#[command(flatten)]
+	target_sign: TargetSigningParams,
+}
+
+/// Messages delivery confirmation relaying params.
+#[derive(Parser)]
+pub struct RelayMessagesDeliveryConfirmationParams {
+	/// Number of the target chain header that we will use to prepare a messages
+	/// delivery proof. This header must be previously proved to the source chain.
+	#[arg(long)]
+	at_target_block: u128,
+	/// Hex-encoded lane id that should be served by the relay.
+	#[arg(long)]
+	lane: HexLaneId,
+	#[command(flatten)]
+	source: SourceConnectionParams,
+	#[command(flatten)]
+	source_sign: SourceSigningParams,
+	#[command(flatten)]
+	target: TargetConnectionParams,
+}
+
+/// Trait used for relaying messages between 2 chains.
+#[async_trait]
+pub trait MessagesRelayer: MessagesCliBridge
+where
+	Self::Source: ChainWithTransactions + ChainWithRuntimeVersion,
+	AccountIdOf<Self::Source>: From<<AccountKeyPairOf<Self::Source> as Pair>::Public>,
+	AccountIdOf<Self::Target>: From<<AccountKeyPairOf<Self::Target> as Pair>::Public>,
+	BalanceOf<Self::Source>: TryFrom<BalanceOf<Self::Target>>,
+{
+	/// Start relaying messages.
+	async fn relay_messages(data: RelayMessagesParams) -> anyhow::Result<()> {
+		let source_client = data.source.into_client::<Self::Source>().await?;
+		let source_sign = data.source_sign.to_keypair::<Self::Source>()?;
+		let source_transactions_mortality = data.source_sign.transactions_mortality()?;
+		let target_client = data.target.into_client::<Self::Target>().await?;
+		let target_sign = data.target_sign.to_keypair::<Self::Target>()?;
+		let target_transactions_mortality = data.target_sign.transactions_mortality()?;
+		let lane_id = HexLaneId::try_convert(data.lane).map_err(|invalid_lane_id| {
+			anyhow::format_err!("Invalid laneId: {:?}!", invalid_lane_id)
+		})?;
+
+		Self::start_relay_guards(&target_client, target_client.can_start_version_guard()).await?;
+
+		crate::messages::run::<Self::MessagesLane, _, _>(MessagesRelayParams {
+			source_client,
+			source_transaction_params: TransactionParams {
+				signer: source_sign,
+				mortality: source_transactions_mortality,
+			},
+			target_client,
+			target_transaction_params: TransactionParams {
+				signer: target_sign,
+				mortality: target_transactions_mortality,
+			},
+			source_to_target_headers_relay: None,
+			target_to_source_headers_relay: None,
+			lane_id,
+			limits: Self::maybe_messages_limits(),
+			metrics_params: data.prometheus_params.into_metrics_params()?,
+		})
+		.await
+		.map_err(|e| anyhow::format_err!("{}", e))
+	}
+
+	/// Relay a consequitive range of messages.
+	async fn relay_messages_range(data: RelayMessagesRangeParams) -> anyhow::Result<()> {
+		let source_client = data.source.into_client::<Self::Source>().await?;
+		let target_client = data.target.into_client::<Self::Target>().await?;
+		let source_sign = data.source_sign.to_keypair::<Self::Source>()?;
+		let source_transactions_mortality = data.source_sign.transactions_mortality()?;
+		let target_sign = data.target_sign.to_keypair::<Self::Target>()?;
+		let target_transactions_mortality = data.target_sign.transactions_mortality()?;
+		let lane_id = HexLaneId::try_convert(data.lane).map_err(|invalid_lane_id| {
+			anyhow::format_err!("Invalid laneId: {:?}!", invalid_lane_id)
+		})?;
+
+		let at_source_block = source_client
+			.header_by_number(data.at_source_block.unique_saturated_into())
+			.await
+			.map_err(|e| {
+				tracing::trace!(
+					target: "bridge",
+					error=?e,
+					source=%Self::Source::NAME,
+					at_source_block=%data.at_source_block,
+					"Failed to read header"
+				);
+				anyhow::format_err!("The command has failed")
+			})?
+			.id();
+
+		crate::messages::relay_messages_range::<Self::MessagesLane>(
+			source_client,
+			target_client,
+			TransactionParams { signer: source_sign, mortality: source_transactions_mortality },
+			TransactionParams { signer: target_sign, mortality: target_transactions_mortality },
+			at_source_block,
+			lane_id,
+			data.messages_start..=data.messages_end,
+			data.outbound_state_proof_required,
+		)
+		.await
+	}
+
+	/// Relay a messages delivery confirmation.
+	async fn relay_messages_delivery_confirmation(
+		data: RelayMessagesDeliveryConfirmationParams,
+	) -> anyhow::Result<()> {
+		let source_client = data.source.into_client::<Self::Source>().await?;
+		let target_client = data.target.into_client::<Self::Target>().await?;
+		let source_sign = data.source_sign.to_keypair::<Self::Source>()?;
+		let source_transactions_mortality = data.source_sign.transactions_mortality()?;
+		let lane_id = HexLaneId::try_convert(data.lane).map_err(|invalid_lane_id| {
+			anyhow::format_err!("Invalid laneId: {:?}!", invalid_lane_id)
+		})?;
+
+		let at_target_block = target_client
+			.header_by_number(data.at_target_block.unique_saturated_into())
+			.await
+			.map_err(|e| {
+				tracing::trace!(
+					target: "bridge",
+					error=?e,
+					target=%Self::Target::NAME,
+					at_target_block=%data.at_target_block,
+					"Failed to read header"
+				);
+				anyhow::format_err!("The command has failed")
+			})?
+			.id();
+
+		crate::messages::relay_messages_delivery_confirmation::<Self::MessagesLane>(
+			source_client,
+			target_client,
+			TransactionParams { signer: source_sign, mortality: source_transactions_mortality },
+			at_target_block,
+			lane_id,
+		)
+		.await
+	}
+
+	/// Add relay guards if required.
+	async fn start_relay_guards(
+		target_client: &impl Client<Self::Target>,
+		enable_version_guard: bool,
+	) -> relay_bizinikiwi_client::Result<()> {
+		if enable_version_guard {
+			relay_bizinikiwi_client::guard::abort_on_spec_version_change(
+				target_client.clone(),
+				target_client.simple_runtime_version().await?.spec_version,
+			);
+		}
+		Ok(())
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/cli/relay_teyrchains.rs

@@ -0,0 +1,158 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Primitives for exposing the teyrchains finality relaying functionality in the CLI.
+
+use async_std::sync::Mutex;
+use async_trait::async_trait;
+use bp_pezkuwi_core::BlockNumber as RelayBlockNumber;
+use pezbp_runtime::HeaderIdProvider;
+use clap::Parser;
+use relay_bizinikiwi_client::{Client, Teyrchain};
+use relay_utils::metrics::{GlobalMetrics, StandaloneMetric};
+use std::sync::Arc;
+use teyrchains_relay::teyrchains_loop::{AvailableHeader, SourceClient, TargetClient};
+
+use crate::{
+	cli::{
+		bridge::{CliBridgeBase, TeyrchainToRelayHeadersCliBridge},
+		chain_schema::*,
+		DefaultClient, PrometheusParams,
+	},
+	finality::BizinikiwiFinalitySyncPipeline,
+	teyrchains::{source::TeyrchainsSource, target::TeyrchainsTarget, TeyrchainsPipelineAdapter},
+	TransactionParams,
+};
+
+/// Teyrchains heads relaying params.
+#[derive(Parser)]
+pub struct RelayTeyrchainsParams {
+	#[command(flatten)]
+	source: SourceConnectionParams,
+	#[command(flatten)]
+	target: TargetConnectionParams,
+	#[command(flatten)]
+	target_sign: TargetSigningParams,
+	/// If passed, only free headers (those, available at "free" relay chain headers)
+	/// are relayed.
+	#[arg(long)]
+	only_free_headers: bool,
+	#[command(flatten)]
+	prometheus_params: PrometheusParams,
+}
+
+/// Single teyrchains head relaying params.
+#[derive(Parser)]
+pub struct RelayTeyrchainHeadParams {
+	#[command(flatten)]
+	source: SourceConnectionParams,
+	#[command(flatten)]
+	target: TargetConnectionParams,
+	#[command(flatten)]
+	target_sign: TargetSigningParams,
+	/// Prove teyrchain head at that relay block number. This relay header must be previously
+	/// proved to the target chain.
+	#[arg(long)]
+	at_relay_block: RelayBlockNumber,
+}
+
+/// Trait used for relaying teyrchains finality between 2 chains.
+#[async_trait]
+pub trait TeyrchainsRelayer: TeyrchainToRelayHeadersCliBridge
+where
+	TeyrchainsSource<Self::TeyrchainFinality, DefaultClient<Self::SourceRelay>>:
+		SourceClient<TeyrchainsPipelineAdapter<Self::TeyrchainFinality>>,
+	TeyrchainsTarget<
+		Self::TeyrchainFinality,
+		DefaultClient<Self::SourceRelay>,
+		DefaultClient<Self::Target>,
+	>: TargetClient<TeyrchainsPipelineAdapter<Self::TeyrchainFinality>>,
+	<Self as CliBridgeBase>::Source: Teyrchain,
+{
+	/// Start relaying teyrchains finality.
+	async fn relay_teyrchains(data: RelayTeyrchainsParams) -> anyhow::Result<()> {
+		let source_chain_client = data.source.into_client::<Self::SourceRelay>().await?;
+		let source_client = TeyrchainsSource::<Self::TeyrchainFinality, _>::new(
+			source_chain_client.clone(),
+			Arc::new(Mutex::new(AvailableHeader::Missing)),
+		);
+
+		let target_transaction_params = TransactionParams {
+			signer: data.target_sign.to_keypair::<Self::Target>()?,
+			mortality: data.target_sign.target_transactions_mortality,
+		};
+		let target_chain_client = data.target.into_client::<Self::Target>().await?;
+		let target_client = TeyrchainsTarget::<Self::TeyrchainFinality, _, _>::new(
+			source_chain_client,
+			target_chain_client,
+			target_transaction_params,
+		);
+
+		let metrics_params: relay_utils::metrics::MetricsParams =
+			data.prometheus_params.into_metrics_params()?;
+		GlobalMetrics::new()?.register_and_spawn(&metrics_params.registry)?;
+
+		Self::RelayFinality::start_relay_guards(
+			target_client.target_client(),
+			target_client.target_client().can_start_version_guard(),
+		)
+		.await?;
+
+		teyrchains_relay::teyrchains_loop::run(
+			source_client,
+			target_client,
+			metrics_params,
+			data.only_free_headers,
+			futures::future::pending(),
+		)
+		.await
+		.map_err(|e| anyhow::format_err!("{}", e))
+	}
+
+	/// Relay single teyrchain head. No checks are made to ensure that transaction will succeed.
+	async fn relay_teyrchain_head(data: RelayTeyrchainHeadParams) -> anyhow::Result<()> {
+		let source_chain_client = data.source.into_client::<Self::SourceRelay>().await?;
+		let at_relay_block = source_chain_client
+			.header_by_number(data.at_relay_block)
+			.await
+			.map_err(|e| anyhow::format_err!("{}", e))?
+			.id();
+
+		let source_client = TeyrchainsSource::<Self::TeyrchainFinality, _>::new(
+			source_chain_client.clone(),
+			Arc::new(Mutex::new(AvailableHeader::Missing)),
+		);
+
+		let target_transaction_params = TransactionParams {
+			signer: data.target_sign.to_keypair::<Self::Target>()?,
+			mortality: data.target_sign.target_transactions_mortality,
+		};
+		let target_chain_client = data.target.into_client::<Self::Target>().await?;
+		let target_client = TeyrchainsTarget::<Self::TeyrchainFinality, _, _>::new(
+			source_chain_client,
+			target_chain_client,
+			target_transaction_params,
+		);
+
+		teyrchains_relay::teyrchains_loop::relay_single_head(
+			source_client,
+			target_client,
+			at_relay_block,
+		)
+		.await
+		.map_err(|_| anyhow::format_err!("The command has failed"))
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/equivocation/mod.rs

@@ -0,0 +1,223 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Types and functions intended to ease adding of new Bizinikiwi -> Bizinikiwi
+//! equivocation detection pipelines.
+
+mod source;
+mod target;
+
+use crate::{
+	equivocation::{source::BizinikiwiEquivocationSource, target::BizinikiwiEquivocationTarget},
+	finality_base::{engine::Engine, BizinikiwiFinalityPipeline, BizinikiwiFinalityProof},
+	TransactionParams,
+};
+
+use async_trait::async_trait;
+use pezbp_runtime::{AccountIdOf, BlockNumberOf, HashOf};
+use pez_equivocation_detector::EquivocationDetectionPipeline;
+use pez_finality_relay::FinalityPipeline;
+use pezpallet_grandpa::{Call as GrandpaCall, Config as GrandpaConfig};
+use relay_bizinikiwi_client::{AccountKeyPairOf, CallOf, Chain, ChainWithTransactions, Client};
+use relay_utils::metrics::MetricsParams;
+use pezsp_core::Pair;
+use pezsp_runtime::traits::{Block, Header};
+use std::marker::PhantomData;
+
+/// Convenience trait that adds bounds to `BizinikiwiEquivocationDetectionPipeline`.
+pub trait BaseBizinikiwiEquivocationDetectionPipeline:
+	BizinikiwiFinalityPipeline<SourceChain = Self::BoundedSourceChain>
+{
+	/// Bounded `BizinikiwiFinalityPipeline::SourceChain`.
+	type BoundedSourceChain: ChainWithTransactions<AccountId = Self::BoundedSourceChainAccountId>;
+
+	/// Bounded `AccountIdOf<BizinikiwiFinalityPipeline::SourceChain>`.
+	type BoundedSourceChainAccountId: From<<AccountKeyPairOf<Self::BoundedSourceChain> as Pair>::Public>
+		+ Send;
+}
+
+impl<T> BaseBizinikiwiEquivocationDetectionPipeline for T
+where
+	T: BizinikiwiFinalityPipeline,
+	T::SourceChain: ChainWithTransactions,
+	AccountIdOf<T::SourceChain>: From<<AccountKeyPairOf<Self::SourceChain> as Pair>::Public>,
+{
+	type BoundedSourceChain = T::SourceChain;
+	type BoundedSourceChainAccountId = AccountIdOf<T::SourceChain>;
+}
+
+/// Bizinikiwi -> Bizinikiwi equivocation detection pipeline.
+#[async_trait]
+pub trait BizinikiwiEquivocationDetectionPipeline:
+	BaseBizinikiwiEquivocationDetectionPipeline
+{
+	/// How the `report_equivocation` call is built ?
+	type ReportEquivocationCallBuilder: ReportEquivocationCallBuilder<Self>;
+
+	/// Add relay guards if required.
+	async fn start_relay_guards(
+		source_client: &impl Client<Self::SourceChain>,
+		enable_version_guard: bool,
+	) -> relay_bizinikiwi_client::Result<()> {
+		if enable_version_guard {
+			relay_bizinikiwi_client::guard::abort_on_spec_version_change(
+				source_client.clone(),
+				source_client.simple_runtime_version().await?.spec_version,
+			);
+		}
+		Ok(())
+	}
+}
+
+type FinalityProoffOf<P> = <<P as BizinikiwiFinalityPipeline>::FinalityEngine as Engine<
+	<P as BizinikiwiFinalityPipeline>::SourceChain,
+>>::FinalityProof;
+type FinalityVerificationContextfOf<P> =
+	<<P as BizinikiwiFinalityPipeline>::FinalityEngine as Engine<
+		<P as BizinikiwiFinalityPipeline>::SourceChain,
+	>>::FinalityVerificationContext;
+/// The type of the equivocation proof used by the `BizinikiwiEquivocationDetectionPipeline`
+pub type EquivocationProofOf<P> = <<P as BizinikiwiFinalityPipeline>::FinalityEngine as Engine<
+	<P as BizinikiwiFinalityPipeline>::SourceChain,
+>>::EquivocationProof;
+type EquivocationsFinderOf<P> = <<P as BizinikiwiFinalityPipeline>::FinalityEngine as Engine<
+	<P as BizinikiwiFinalityPipeline>::SourceChain,
+>>::EquivocationsFinder;
+/// The type of the key owner proof used by the `BizinikiwiEquivocationDetectionPipeline`
+pub type KeyOwnerProofOf<P> = <<P as BizinikiwiFinalityPipeline>::FinalityEngine as Engine<
+	<P as BizinikiwiFinalityPipeline>::SourceChain,
+>>::KeyOwnerProof;
+
+/// Adapter that allows a `BizinikiwiEquivocationDetectionPipeline` to act as an
+/// `EquivocationDetectionPipeline`.
+#[derive(Clone, Debug)]
+pub struct EquivocationDetectionPipelineAdapter<P: BizinikiwiEquivocationDetectionPipeline> {
+	_phantom: PhantomData<P>,
+}
+
+impl<P: BizinikiwiEquivocationDetectionPipeline> FinalityPipeline
+	for EquivocationDetectionPipelineAdapter<P>
+{
+	const SOURCE_NAME: &'static str = P::SourceChain::NAME;
+	const TARGET_NAME: &'static str = P::TargetChain::NAME;
+
+	type Hash = HashOf<P::SourceChain>;
+	type Number = BlockNumberOf<P::SourceChain>;
+	type FinalityProof = BizinikiwiFinalityProof<P>;
+}
+
+impl<P: BizinikiwiEquivocationDetectionPipeline> EquivocationDetectionPipeline
+	for EquivocationDetectionPipelineAdapter<P>
+{
+	type TargetNumber = BlockNumberOf<P::TargetChain>;
+	type FinalityVerificationContext = FinalityVerificationContextfOf<P>;
+	type EquivocationProof = EquivocationProofOf<P>;
+	type EquivocationsFinder = EquivocationsFinderOf<P>;
+}
+
+/// Different ways of building `report_equivocation` calls.
+pub trait ReportEquivocationCallBuilder<P: BizinikiwiEquivocationDetectionPipeline> {
+	/// Build a `report_equivocation` call to be executed on the source chain.
+	fn build_report_equivocation_call(
+		equivocation_proof: EquivocationProofOf<P>,
+		key_owner_proof: KeyOwnerProofOf<P>,
+	) -> CallOf<P::SourceChain>;
+}
+
+/// Building the `report_equivocation` call when having direct access to the target chain runtime.
+pub struct DirectReportGrandpaEquivocationCallBuilder<P, R> {
+	_phantom: PhantomData<(P, R)>,
+}
+
+impl<P, R> ReportEquivocationCallBuilder<P> for DirectReportGrandpaEquivocationCallBuilder<P, R>
+where
+	P: BizinikiwiEquivocationDetectionPipeline,
+	P::FinalityEngine: Engine<
+		P::SourceChain,
+		EquivocationProof = pezsp_consensus_grandpa::EquivocationProof<
+			HashOf<P::SourceChain>,
+			BlockNumberOf<P::SourceChain>,
+		>,
+	>,
+	R: pezframe_system::Config<Hash = HashOf<P::SourceChain>>
+		+ GrandpaConfig<KeyOwnerProof = KeyOwnerProofOf<P>>,
+	<R::Block as Block>::Header: Header<Number = BlockNumberOf<P::SourceChain>>,
+	CallOf<P::SourceChain>: From<GrandpaCall<R>>,
+{
+	fn build_report_equivocation_call(
+		equivocation_proof: EquivocationProofOf<P>,
+		key_owner_proof: KeyOwnerProofOf<P>,
+	) -> CallOf<P::SourceChain> {
+		GrandpaCall::<R>::report_equivocation {
+			equivocation_proof: Box::new(equivocation_proof),
+			key_owner_proof,
+		}
+		.into()
+	}
+}
+
+/// Macro that generates `ReportEquivocationCallBuilder` implementation for the case where
+/// we only have access to the mocked version of the source chain runtime.
+#[rustfmt::skip]
+#[macro_export]
+macro_rules! generate_report_equivocation_call_builder {
+	($pipeline:ident, $mocked_builder:ident, $grandpa:path, $report_equivocation:path) => {
+		pub struct $mocked_builder;
+
+		impl $crate::equivocation::ReportEquivocationCallBuilder<$pipeline>
+			for $mocked_builder
+		{
+			fn build_report_equivocation_call(
+				equivocation_proof: $crate::equivocation::EquivocationProofOf<$pipeline>,
+				key_owner_proof: $crate::equivocation::KeyOwnerProofOf<$pipeline>,
+			) -> relay_bizinikiwi_client::CallOf<
+				<$pipeline as $crate::finality_base::BizinikiwiFinalityPipeline>::SourceChain
+			> {
+				pezbp_runtime::paste::item! {
+					$grandpa($report_equivocation {
+						equivocation_proof: Box::new(equivocation_proof),
+						key_owner_proof: key_owner_proof
+					})
+				}
+			}
+		}
+	};
+}
+
+/// Run Bizinikiwi-to-Bizinikiwi equivocations detection loop.
+pub async fn run<P: BizinikiwiEquivocationDetectionPipeline>(
+	source_client: impl Client<P::SourceChain>,
+	target_client: impl Client<P::TargetChain>,
+	source_transaction_params: TransactionParams<AccountKeyPairOf<P::SourceChain>>,
+	metrics_params: MetricsParams,
+) -> anyhow::Result<()> {
+	tracing::info!(
+		target: "bridge",
+		source=%P::SourceChain::NAME,
+		target=%P::TargetChain::NAME,
+		"Starting equivocations detection loop"
+	);
+
+	pez_equivocation_detector::run(
+		BizinikiwiEquivocationSource::<P, _>::new(source_client, source_transaction_params),
+		BizinikiwiEquivocationTarget::<P, _>::new(target_client),
+		P::TargetChain::AVERAGE_BLOCK_INTERVAL,
+		metrics_params,
+		futures::future::pending(),
+	)
+	.await
+	.map_err(|e| anyhow::format_err!("{}", e))
+}

pezbridges/relays/lib-bizinikiwi-relay/src/equivocation/source.rs

@@ -0,0 +1,117 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Default generic implementation of equivocation source for basic Bizinikiwi client.
+
+use crate::{
+	equivocation::{
+		EquivocationDetectionPipelineAdapter, EquivocationProofOf, ReportEquivocationCallBuilder,
+		BizinikiwiEquivocationDetectionPipeline,
+	},
+	finality_base::{engine::Engine, finality_proofs, BizinikiwiFinalityProofsStream},
+	TransactionParams,
+};
+
+use async_trait::async_trait;
+use pezbp_runtime::{HashOf, TransactionEra};
+use pez_equivocation_detector::SourceClient;
+use pez_finality_relay::SourceClientBase;
+use relay_bizinikiwi_client::{
+	AccountKeyPairOf, Client, Error, TransactionTracker, UnsignedTransaction,
+};
+use relay_utils::relay_loop::Client as RelayClient;
+
+/// Bizinikiwi node as equivocation source.
+pub struct BizinikiwiEquivocationSource<P: BizinikiwiEquivocationDetectionPipeline, SourceClnt> {
+	client: SourceClnt,
+	transaction_params: TransactionParams<AccountKeyPairOf<P::SourceChain>>,
+}
+
+impl<P: BizinikiwiEquivocationDetectionPipeline, SourceClnt: Client<P::SourceChain>>
+	BizinikiwiEquivocationSource<P, SourceClnt>
+{
+	/// Create new instance of `BizinikiwiEquivocationSource`.
+	pub fn new(
+		client: SourceClnt,
+		transaction_params: TransactionParams<AccountKeyPairOf<P::SourceChain>>,
+	) -> Self {
+		Self { client, transaction_params }
+	}
+}
+
+impl<P: BizinikiwiEquivocationDetectionPipeline, SourceClnt: Client<P::SourceChain>> Clone
+	for BizinikiwiEquivocationSource<P, SourceClnt>
+{
+	fn clone(&self) -> Self {
+		Self { client: self.client.clone(), transaction_params: self.transaction_params.clone() }
+	}
+}
+
+#[async_trait]
+impl<P: BizinikiwiEquivocationDetectionPipeline, SourceClnt: Client<P::SourceChain>> RelayClient
+	for BizinikiwiEquivocationSource<P, SourceClnt>
+{
+	type Error = Error;
+
+	async fn reconnect(&mut self) -> Result<(), Error> {
+		self.client.reconnect().await
+	}
+}
+
+#[async_trait]
+impl<P: BizinikiwiEquivocationDetectionPipeline, SourceClnt: Client<P::SourceChain>>
+	SourceClientBase<EquivocationDetectionPipelineAdapter<P>>
+	for BizinikiwiEquivocationSource<P, SourceClnt>
+{
+	type FinalityProofsStream = BizinikiwiFinalityProofsStream<P>;
+
+	async fn finality_proofs(&self) -> Result<Self::FinalityProofsStream, Error> {
+		finality_proofs::<P>(&self.client).await
+	}
+}
+
+#[async_trait]
+impl<P: BizinikiwiEquivocationDetectionPipeline, SourceClnt: Client<P::SourceChain>>
+	SourceClient<EquivocationDetectionPipelineAdapter<P>>
+	for BizinikiwiEquivocationSource<P, SourceClnt>
+{
+	type TransactionTracker = TransactionTracker<P::SourceChain, SourceClnt>;
+
+	async fn report_equivocation(
+		&self,
+		at: HashOf<P::SourceChain>,
+		equivocation: EquivocationProofOf<P>,
+	) -> Result<Self::TransactionTracker, Self::Error> {
+		let key_owner_proof =
+			P::FinalityEngine::generate_source_key_ownership_proof(&self.client, at, &equivocation)
+				.await?;
+
+		let mortality = self.transaction_params.mortality;
+		let call = P::ReportEquivocationCallBuilder::build_report_equivocation_call(
+			equivocation,
+			key_owner_proof,
+		);
+		self.client
+			.submit_and_watch_signed_extrinsic(
+				&self.transaction_params.signer,
+				move |best_block_id, transaction_nonce| {
+					Ok(UnsignedTransaction::new(call.into(), transaction_nonce)
+						.era(TransactionEra::new(best_block_id, mortality)))
+				},
+			)
+			.await
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/equivocation/target.rs

@@ -0,0 +1,118 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Default generic implementation of equivocation source for basic Bizinikiwi client.
+
+use crate::{
+	equivocation::{
+		EquivocationDetectionPipelineAdapter, FinalityProoffOf, FinalityVerificationContextfOf,
+		BizinikiwiEquivocationDetectionPipeline,
+	},
+	finality_base::{best_synced_header_id, engine::Engine},
+};
+
+use async_trait::async_trait;
+use bp_header_pez_chain::HeaderFinalityInfo;
+use pezbp_runtime::{BlockNumberOf, HashOf};
+use pez_equivocation_detector::TargetClient;
+use relay_bizinikiwi_client::{Client, Error};
+use relay_utils::relay_loop::Client as RelayClient;
+use pezsp_runtime::traits::Header;
+use std::marker::PhantomData;
+
+/// Bizinikiwi node as equivocation source.
+pub struct BizinikiwiEquivocationTarget<P: BizinikiwiEquivocationDetectionPipeline, TargetClnt> {
+	client: TargetClnt,
+
+	_phantom: PhantomData<P>,
+}
+
+impl<P: BizinikiwiEquivocationDetectionPipeline, TargetClnt: Client<P::TargetChain>>
+	BizinikiwiEquivocationTarget<P, TargetClnt>
+{
+	/// Create new instance of `BizinikiwiEquivocationTarget`.
+	pub fn new(client: TargetClnt) -> Self {
+		Self { client, _phantom: Default::default() }
+	}
+}
+
+impl<P: BizinikiwiEquivocationDetectionPipeline, TargetClnt: Client<P::TargetChain>> Clone
+	for BizinikiwiEquivocationTarget<P, TargetClnt>
+{
+	fn clone(&self) -> Self {
+		Self { client: self.client.clone(), _phantom: Default::default() }
+	}
+}
+
+#[async_trait]
+impl<P: BizinikiwiEquivocationDetectionPipeline, TargetClnt: Client<P::TargetChain>> RelayClient
+	for BizinikiwiEquivocationTarget<P, TargetClnt>
+{
+	type Error = Error;
+
+	async fn reconnect(&mut self) -> Result<(), Error> {
+		self.client.reconnect().await
+	}
+}
+
+#[async_trait]
+impl<P: BizinikiwiEquivocationDetectionPipeline, TargetClnt: Client<P::TargetChain>>
+	TargetClient<EquivocationDetectionPipelineAdapter<P>>
+	for BizinikiwiEquivocationTarget<P, TargetClnt>
+{
+	async fn best_finalized_header_number(
+		&self,
+	) -> Result<BlockNumberOf<P::TargetChain>, Self::Error> {
+		self.client.best_finalized_header_number().await
+	}
+
+	async fn best_synced_header_hash(
+		&self,
+		at: BlockNumberOf<P::TargetChain>,
+	) -> Result<Option<HashOf<P::SourceChain>>, Self::Error> {
+		Ok(best_synced_header_id::<P::SourceChain, P::TargetChain>(
+			&self.client,
+			self.client.header_by_number(at).await?.hash(),
+		)
+		.await?
+		.map(|id| id.hash()))
+	}
+
+	async fn finality_verification_context(
+		&self,
+		at: BlockNumberOf<P::TargetChain>,
+	) -> Result<FinalityVerificationContextfOf<P>, Self::Error> {
+		P::FinalityEngine::finality_verification_context(
+			&self.client,
+			self.client.header_by_number(at).await?.hash(),
+		)
+		.await
+	}
+
+	async fn synced_headers_finality_info(
+		&self,
+		at: BlockNumberOf<P::TargetChain>,
+	) -> Result<
+		Vec<HeaderFinalityInfo<FinalityProoffOf<P>, FinalityVerificationContextfOf<P>>>,
+		Self::Error,
+	> {
+		P::FinalityEngine::synced_headers_finality_info(
+			&self.client,
+			self.client.header_by_number(at).await?.hash(),
+		)
+		.await
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/error.rs

@@ -0,0 +1,63 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Relay errors.
+
+use relay_bizinikiwi_client as client;
+use pezsp_consensus_grandpa::AuthorityList;
+use pezsp_runtime::traits::MaybeDisplay;
+use std::fmt::Debug;
+use thiserror::Error;
+
+/// Relay errors.
+#[derive(Error, Debug)]
+pub enum Error<Hash: Debug + MaybeDisplay, HeaderNumber: Debug + MaybeDisplay> {
+	/// Failed to submit signed extrinsic from to the target chain.
+	#[error("Failed to submit {0} transaction: {1:?}")]
+	SubmitTransaction(&'static str, client::Error),
+	/// Failed subscribe to justification stream of the source chain.
+	#[error("Failed to subscribe to {0} justifications: {1:?}")]
+	Subscribe(&'static str, client::Error),
+	/// Failed subscribe to read justification from the source chain (client error).
+	#[error("Failed to read {0} justification from the stream: {1}")]
+	ReadJustification(&'static str, client::Error),
+	/// Failed subscribe to read justification from the source chain (stream ended).
+	#[error("Failed to read {0} justification from the stream: stream has ended unexpectedly")]
+	ReadJustificationStreamEnded(&'static str),
+	/// Failed subscribe to decode justification from the source chain.
+	#[error("Failed to decode {0} justification: {1:?}")]
+	DecodeJustification(&'static str, codec::Error),
+	/// GRANDPA authorities read from the source chain are invalid.
+	#[error("Read invalid {0} authorities set: {1:?}")]
+	ReadInvalidAuthorities(&'static str, AuthorityList),
+	/// Failed to guess initial GRANDPA authorities at the given header of the source chain.
+	#[error("Failed to guess initial {0} GRANDPA authorities set id: checked all possible ids in range [0; {1}]")]
+	GuessInitialAuthorities(&'static str, HeaderNumber),
+	/// Failed to retrieve GRANDPA authorities at the given header from the source chain.
+	#[error("Failed to retrieve {0} GRANDPA authorities set at header {1}: {2:?}")]
+	RetrieveAuthorities(&'static str, Hash, client::Error),
+	/// Failed to decode GRANDPA authorities at the given header of the source chain.
+	#[error("Failed to decode {0} GRANDPA authorities set at header {1}: {2:?}")]
+	DecodeAuthorities(&'static str, Hash, codec::Error),
+	/// Failed to retrieve header by the hash from the source chain.
+	#[error("Failed to retrieve {0} header with hash {1}: {2:?}")]
+	RetrieveHeader(&'static str, Hash, client::Error),
+	/// Failed to submit signed extrinsic from to the target chain.
+	#[error(
+		"Failed to retrieve `is_initialized` flag of the with-{0} finality pezpallet at {1}: {2:?}"
+	)]
+	IsInitializedRetrieve(&'static str, &'static str, client::Error),
+}

pezbridges/relays/lib-bizinikiwi-relay/src/finality/initialize.rs

@@ -0,0 +1,163 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Initialize Bizinikiwi -> Bizinikiwi finality bridge.
+//!
+//! Initialization is a transaction that calls `initialize()` function of the
+//! finality pezpallet (GRANDPA/BEEFY/...). This transaction brings initial header
+//! and authorities set from source to target chain. The finality sync starts
+//! with this header.
+
+use crate::{error::Error, finality_base::engine::Engine};
+use pezsp_core::Pair;
+
+use pezbp_runtime::HeaderIdOf;
+use relay_bizinikiwi_client::{
+	AccountKeyPairOf, Chain, ChainWithTransactions, Client, Error as BizinikiwiError,
+	UnsignedTransaction,
+};
+use relay_utils::{TrackedTransactionStatus, TransactionTracker};
+use pezsp_runtime::traits::Header as HeaderT;
+
+/// Submit headers-bridge initialization transaction.
+pub async fn initialize<
+	E: Engine<SourceChain>,
+	SourceChain: Chain,
+	TargetChain: ChainWithTransactions,
+	F,
+>(
+	source_client: impl Client<SourceChain>,
+	target_client: impl Client<TargetChain>,
+	target_signer: AccountKeyPairOf<TargetChain>,
+	prepare_initialize_transaction: F,
+	dry_run: bool,
+) where
+	F: FnOnce(
+			TargetChain::Nonce,
+			E::InitializationData,
+		) -> Result<UnsignedTransaction<TargetChain>, BizinikiwiError>
+		+ Send
+		+ 'static,
+	TargetChain::AccountId: From<<TargetChain::AccountKeyPair as Pair>::Public>,
+{
+	let result = do_initialize::<E, _, _, _>(
+		source_client,
+		target_client,
+		target_signer,
+		prepare_initialize_transaction,
+		dry_run,
+	)
+	.await;
+
+	match result {
+		Ok(Some(tx_status)) => match tx_status {
+			TrackedTransactionStatus::Lost => {
+				tracing::error!(
+					target: "bridge",
+					source=%SourceChain::NAME,
+					target=%TargetChain::NAME,
+					?tx_status,
+					"Failed to execute headers bridge initialization transaction."
+				)
+			},
+			TrackedTransactionStatus::Finalized(_) => {
+				tracing::info!(
+					target: "bridge",
+					source=%SourceChain::NAME,
+					target=%TargetChain::NAME,
+					?tx_status,
+					"Successfully executed headers bridge initialization transaction."
+				)
+			},
+		},
+		Ok(None) => (),
+		Err(err) => tracing::error!(
+			target: "bridge",
+			error=?err,
+			source=%SourceChain::NAME,
+			target=%TargetChain::NAME,
+			"Failed to submit headers bridge initialization transaction"
+		),
+	}
+}
+
+/// Craft and submit initialization transaction, returning any error that may occur.
+async fn do_initialize<
+	E: Engine<SourceChain>,
+	SourceChain: Chain,
+	TargetChain: ChainWithTransactions,
+	F,
+>(
+	source_client: impl Client<SourceChain>,
+	target_client: impl Client<TargetChain>,
+	target_signer: AccountKeyPairOf<TargetChain>,
+	prepare_initialize_transaction: F,
+	dry_run: bool,
+) -> Result<
+	Option<TrackedTransactionStatus<HeaderIdOf<TargetChain>>>,
+	Error<SourceChain::Hash, <SourceChain::Header as HeaderT>::Number>,
+>
+where
+	F: FnOnce(
+			TargetChain::Nonce,
+			E::InitializationData,
+		) -> Result<UnsignedTransaction<TargetChain>, BizinikiwiError>
+		+ Send
+		+ 'static,
+	TargetChain::AccountId: From<<TargetChain::AccountKeyPair as Pair>::Public>,
+{
+	let is_initialized = E::is_initialized(&target_client)
+		.await
+		.map_err(|e| Error::IsInitializedRetrieve(SourceChain::NAME, TargetChain::NAME, e))?;
+	if is_initialized {
+		tracing::info!(
+			target: "bridge",
+			source=%SourceChain::NAME,
+			target=%TargetChain::NAME,
+			"Headers bridge is already initialized. Skipping"
+		);
+		if !dry_run {
+			return Ok(None);
+		}
+	}
+
+	let initialization_data = E::prepare_initialization_data(source_client).await?;
+	tracing::info!(
+		target: "bridge",
+		source=%SourceChain::NAME,
+		target=%TargetChain::NAME,
+		?initialization_data,
+		"Prepared initialization data for headers bridge"
+	);
+
+	let tx_status = target_client
+		.submit_and_watch_signed_extrinsic(&target_signer, move |_, transaction_nonce| {
+			let tx = prepare_initialize_transaction(transaction_nonce, initialization_data);
+			if dry_run {
+				Err(BizinikiwiError::Custom(
+					"Not submitting extrinsic in `dry-run` mode!".to_string(),
+				))
+			} else {
+				tx
+			}
+		})
+		.await
+		.map_err(|err| Error::SubmitTransaction(TargetChain::NAME, err))?
+		.wait()
+		.await;
+
+	Ok(Some(tx_status))
+}

pezbridges/relays/lib-bizinikiwi-relay/src/finality/mod.rs

@@ -0,0 +1,309 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Types and functions intended to ease adding of new Bizinikiwi -> Bizinikiwi
+//! finality proofs synchronization pipelines.
+
+use crate::{
+	finality::{source::BizinikiwiFinalitySource, target::BizinikiwiFinalityTarget},
+	finality_base::{engine::Engine, BizinikiwiFinalityPipeline, BizinikiwiFinalityProof},
+	TransactionParams,
+};
+
+use async_trait::async_trait;
+use bp_header_pez_chain::justification::{GrandpaJustification, JustificationVerificationContext};
+use pez_finality_relay::{
+	FinalityPipeline, FinalitySyncPipeline, HeadersToRelay, SourceClient, TargetClient,
+};
+use pezpallet_bridge_grandpa::{Call as BridgeGrandpaCall, Config as BridgeGrandpaConfig};
+use relay_bizinikiwi_client::{
+	transaction_stall_timeout, AccountIdOf, AccountKeyPairOf, BlockNumberOf, CallOf, Chain,
+	ChainWithTransactions, Client, HashOf, HeaderOf, SyncHeader,
+};
+use relay_utils::{metrics::MetricsParams, TrackedTransactionStatus, TransactionTracker};
+use pezsp_core::Pair;
+use std::{fmt::Debug, marker::PhantomData};
+
+pub mod initialize;
+pub mod source;
+pub mod target;
+
+/// Default limit of recent finality proofs.
+///
+/// Finality delay of 4096 blocks is unlikely to happen in practice in
+/// Bizinikiwi+GRANDPA based chains (good to know).
+pub(crate) const RECENT_FINALITY_PROOFS_LIMIT: usize = 4096;
+
+/// Convenience trait that adds bounds to `BizinikiwiFinalitySyncPipeline`.
+pub trait BaseBizinikiwiFinalitySyncPipeline:
+	BizinikiwiFinalityPipeline<TargetChain = Self::BoundedTargetChain>
+{
+	/// Bounded `BizinikiwiFinalityPipeline::TargetChain`.
+	type BoundedTargetChain: ChainWithTransactions<AccountId = Self::BoundedTargetChainAccountId>;
+
+	/// Bounded `AccountIdOf<BizinikiwiFinalityPipeline::TargetChain>`.
+	type BoundedTargetChainAccountId: From<<AccountKeyPairOf<Self::BoundedTargetChain> as Pair>::Public>
+		+ Send;
+}
+
+impl<T> BaseBizinikiwiFinalitySyncPipeline for T
+where
+	T: BizinikiwiFinalityPipeline,
+	T::TargetChain: ChainWithTransactions,
+	AccountIdOf<T::TargetChain>: From<<AccountKeyPairOf<Self::TargetChain> as Pair>::Public>,
+{
+	type BoundedTargetChain = T::TargetChain;
+	type BoundedTargetChainAccountId = AccountIdOf<T::TargetChain>;
+}
+
+/// Bizinikiwi -> Bizinikiwi finality proofs synchronization pipeline.
+#[async_trait]
+pub trait BizinikiwiFinalitySyncPipeline: BaseBizinikiwiFinalitySyncPipeline {
+	/// How submit finality proof call is built?
+	type SubmitFinalityProofCallBuilder: SubmitFinalityProofCallBuilder<Self>;
+
+	/// Add relay guards if required.
+	async fn start_relay_guards(
+		target_client: &impl Client<Self::TargetChain>,
+		enable_version_guard: bool,
+	) -> relay_bizinikiwi_client::Result<()> {
+		if enable_version_guard {
+			relay_bizinikiwi_client::guard::abort_on_spec_version_change(
+				target_client.clone(),
+				target_client.simple_runtime_version().await?.spec_version,
+			);
+		}
+		Ok(())
+	}
+}
+
+/// Adapter that allows all `BizinikiwiFinalitySyncPipeline` to act as `FinalitySyncPipeline`.
+#[derive(Clone, Debug)]
+pub struct FinalitySyncPipelineAdapter<P: BizinikiwiFinalitySyncPipeline> {
+	_phantom: PhantomData<P>,
+}
+
+impl<P: BizinikiwiFinalitySyncPipeline> FinalityPipeline for FinalitySyncPipelineAdapter<P> {
+	const SOURCE_NAME: &'static str = P::SourceChain::NAME;
+	const TARGET_NAME: &'static str = P::TargetChain::NAME;
+
+	type Hash = HashOf<P::SourceChain>;
+	type Number = BlockNumberOf<P::SourceChain>;
+	type FinalityProof = BizinikiwiFinalityProof<P>;
+}
+
+impl<P: BizinikiwiFinalitySyncPipeline> FinalitySyncPipeline for FinalitySyncPipelineAdapter<P> {
+	type ConsensusLogReader = <P::FinalityEngine as Engine<P::SourceChain>>::ConsensusLogReader;
+	type Header = SyncHeader<HeaderOf<P::SourceChain>>;
+}
+
+/// Different ways of building `submit_finality_proof` calls.
+pub trait SubmitFinalityProofCallBuilder<P: BizinikiwiFinalitySyncPipeline> {
+	/// Given source chain header, its finality proof and the current authority set id, build call
+	/// of `submit_finality_proof` function of bridge GRANDPA module at the target chain.
+	fn build_submit_finality_proof_call(
+		header: SyncHeader<HeaderOf<P::SourceChain>>,
+		proof: BizinikiwiFinalityProof<P>,
+		is_free_execution_expected: bool,
+		context: <<P as BizinikiwiFinalityPipeline>::FinalityEngine as Engine<P::SourceChain>>::FinalityVerificationContext,
+	) -> CallOf<P::TargetChain>;
+}
+
+/// Building `submit_finality_proof` call when you have direct access to the target
+/// chain runtime.
+pub struct DirectSubmitGrandpaFinalityProofCallBuilder<P, R, I> {
+	_phantom: PhantomData<(P, R, I)>,
+}
+
+impl<P, R, I> SubmitFinalityProofCallBuilder<P>
+	for DirectSubmitGrandpaFinalityProofCallBuilder<P, R, I>
+where
+	P: BizinikiwiFinalitySyncPipeline,
+	R: BridgeGrandpaConfig<I>,
+	I: 'static,
+	R::BridgedChain: pezbp_runtime::Chain<Header = HeaderOf<P::SourceChain>>,
+	CallOf<P::TargetChain>: From<BridgeGrandpaCall<R, I>>,
+	P::FinalityEngine: Engine<
+		P::SourceChain,
+		FinalityProof = GrandpaJustification<HeaderOf<P::SourceChain>>,
+		FinalityVerificationContext = JustificationVerificationContext,
+	>,
+{
+	fn build_submit_finality_proof_call(
+		header: SyncHeader<HeaderOf<P::SourceChain>>,
+		proof: GrandpaJustification<HeaderOf<P::SourceChain>>,
+		_is_free_execution_expected: bool,
+		_context: JustificationVerificationContext,
+	) -> CallOf<P::TargetChain> {
+		BridgeGrandpaCall::<R, I>::submit_finality_proof {
+			finality_target: Box::new(header.into_inner()),
+			justification: proof,
+		}
+		.into()
+	}
+}
+
+/// Macro that generates `SubmitFinalityProofCallBuilder` implementation for the case when
+/// you only have an access to the mocked version of target chain runtime. In this case you
+/// should provide "name" of the call variant for the bridge GRANDPA calls and the "name" of
+/// the variant for the `submit_finality_proof` call within that first option.
+#[rustfmt::skip]
+#[macro_export]
+macro_rules! generate_submit_finality_proof_call_builder {
+	($pipeline:ident, $mocked_builder:ident, $bridge_grandpa:path, $submit_finality_proof:path) => {
+		pub struct $mocked_builder;
+
+		impl $crate::finality::SubmitFinalityProofCallBuilder<$pipeline>
+			for $mocked_builder
+		{
+			fn build_submit_finality_proof_call(
+				header: relay_bizinikiwi_client::SyncHeader<
+					relay_bizinikiwi_client::HeaderOf<
+						<$pipeline as $crate::finality_base::BizinikiwiFinalityPipeline>::SourceChain
+					>
+				>,
+				proof: bp_header_pez_chain::justification::GrandpaJustification<
+					relay_bizinikiwi_client::HeaderOf<
+						<$pipeline as $crate::finality_base::BizinikiwiFinalityPipeline>::SourceChain
+					>
+				>,
+				_is_free_execution_expected: bool,
+				_context: bp_header_pez_chain::justification::JustificationVerificationContext,
+			) -> relay_bizinikiwi_client::CallOf<
+				<$pipeline as $crate::finality_base::BizinikiwiFinalityPipeline>::TargetChain
+			> {
+				pezbp_runtime::paste::item! {
+					$bridge_grandpa($submit_finality_proof {
+						finality_target: Box::new(header.into_inner()),
+						justification: proof
+					})
+				}
+			}
+		}
+	};
+}
+
+/// Macro that generates `SubmitFinalityProofCallBuilder` implementation for the case when
+/// you only have an access to the mocked version of target chain runtime. In this case you
+/// should provide "name" of the call variant for the bridge GRANDPA calls and the "name" of
+/// the variant for the `submit_finality_proof_ex` call within that first option.
+#[rustfmt::skip]
+#[macro_export]
+macro_rules! generate_submit_finality_proof_ex_call_builder {
+	($pipeline:ident, $mocked_builder:ident, $bridge_grandpa:path, $submit_finality_proof:path) => {
+		pub struct $mocked_builder;
+
+		impl $crate::finality::SubmitFinalityProofCallBuilder<$pipeline>
+			for $mocked_builder
+		{
+			fn build_submit_finality_proof_call(
+				header: relay_bizinikiwi_client::SyncHeader<
+					relay_bizinikiwi_client::HeaderOf<
+						<$pipeline as $crate::finality_base::BizinikiwiFinalityPipeline>::SourceChain
+					>
+				>,
+				proof: bp_header_pez_chain::justification::GrandpaJustification<
+					relay_bizinikiwi_client::HeaderOf<
+						<$pipeline as $crate::finality_base::BizinikiwiFinalityPipeline>::SourceChain
+					>
+				>,
+				is_free_execution_expected: bool,
+				context: bp_header_pez_chain::justification::JustificationVerificationContext,
+			) -> relay_bizinikiwi_client::CallOf<
+				<$pipeline as $crate::finality_base::BizinikiwiFinalityPipeline>::TargetChain
+			> {
+				pezbp_runtime::paste::item! {
+					$bridge_grandpa($submit_finality_proof {
+						finality_target: Box::new(header.into_inner()),
+						justification: proof,
+						current_set_id: context.authority_set_id,
+						is_free_execution_expected,
+					})
+				}
+			}
+		}
+	};
+}
+
+/// Run Bizinikiwi-to-Bizinikiwi finality sync loop.
+pub async fn run<P: BizinikiwiFinalitySyncPipeline>(
+	source_client: impl Client<P::SourceChain>,
+	target_client: impl Client<P::TargetChain>,
+	headers_to_relay: HeadersToRelay,
+	transaction_params: TransactionParams<AccountKeyPairOf<P::TargetChain>>,
+	metrics_params: MetricsParams,
+) -> anyhow::Result<()> {
+	tracing::info!(
+		target: "bridge",
+		source=%P::SourceChain::NAME,
+		target=%P::TargetChain::NAME,
+		?headers_to_relay,
+		"Starting source -> target finality proof relay"
+	);
+
+	pez_finality_relay::run(
+		BizinikiwiFinalitySource::<P, _>::new(source_client, None),
+		BizinikiwiFinalityTarget::<P, _>::new(target_client, transaction_params.clone()),
+		pez_finality_relay::FinalitySyncParams {
+			tick: std::cmp::max(
+				P::SourceChain::AVERAGE_BLOCK_INTERVAL,
+				P::TargetChain::AVERAGE_BLOCK_INTERVAL,
+			),
+			recent_finality_proofs_limit: RECENT_FINALITY_PROOFS_LIMIT,
+			stall_timeout: transaction_stall_timeout(
+				transaction_params.mortality,
+				P::TargetChain::AVERAGE_BLOCK_INTERVAL,
+				relay_utils::STALL_TIMEOUT,
+			),
+			headers_to_relay,
+		},
+		metrics_params,
+		futures::future::pending(),
+	)
+	.await
+	.map_err(|e| anyhow::format_err!("{}", e))
+}
+
+/// Relay single header. No checks are made to ensure that transaction will succeed.
+pub async fn relay_single_header<P: BizinikiwiFinalitySyncPipeline>(
+	source_client: impl Client<P::SourceChain>,
+	target_client: impl Client<P::TargetChain>,
+	transaction_params: TransactionParams<AccountKeyPairOf<P::TargetChain>>,
+	header_number: BlockNumberOf<P::SourceChain>,
+) -> anyhow::Result<()> {
+	let finality_source = BizinikiwiFinalitySource::<P, _>::new(source_client, None);
+	let (header, proof) = finality_source.header_and_finality_proof(header_number).await?;
+	let Some(proof) = proof else {
+		return Err(anyhow::format_err!(
+			"Unable to submit {} header #{} to {}: no finality proof",
+			P::SourceChain::NAME,
+			header_number,
+			P::TargetChain::NAME,
+		));
+	};
+
+	let finality_target = BizinikiwiFinalityTarget::<P, _>::new(target_client, transaction_params);
+	let tx_tracker = finality_target.submit_finality_proof(header, proof, false).await?;
+	match tx_tracker.wait().await {
+		TrackedTransactionStatus::Finalized(_) => Ok(()),
+		TrackedTransactionStatus::Lost => Err(anyhow::format_err!(
+			"Transaction with {} header #{} is considered lost at {}",
+			P::SourceChain::NAME,
+			header_number,
+			P::TargetChain::NAME,
+		)),
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/finality/source.rs

@@ -0,0 +1,265 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Default generic implementation of finality source for basic Bizinikiwi client.
+
+use crate::{
+	finality::{FinalitySyncPipelineAdapter, BizinikiwiFinalitySyncPipeline},
+	finality_base::{
+		engine::Engine, finality_proofs, BizinikiwiFinalityProof, BizinikiwiFinalityProofsStream,
+	},
+};
+
+use async_std::sync::{Arc, Mutex};
+use async_trait::async_trait;
+use bp_header_pez_chain::FinalityProof;
+use codec::Decode;
+use pez_finality_relay::{SourceClient, SourceClientBase};
+use futures::{
+	select,
+	stream::{try_unfold, Stream, StreamExt, TryStreamExt},
+};
+use num_traits::One;
+use relay_bizinikiwi_client::{BlockNumberOf, BlockWithJustification, Client, Error, HeaderOf};
+use relay_utils::{relay_loop::Client as RelayClient, UniqueSaturatedInto};
+
+/// Shared updatable reference to the maximal header number that we want to sync from the source.
+pub type RequiredHeaderNumberRef<C> = Arc<Mutex<<C as pezbp_runtime::Chain>::BlockNumber>>;
+
+/// Bizinikiwi node as finality source.
+pub struct BizinikiwiFinalitySource<P: BizinikiwiFinalitySyncPipeline, SourceClnt> {
+	client: SourceClnt,
+	maximal_header_number: Option<RequiredHeaderNumberRef<P::SourceChain>>,
+}
+
+impl<P: BizinikiwiFinalitySyncPipeline, SourceClnt: Client<P::SourceChain>>
+	BizinikiwiFinalitySource<P, SourceClnt>
+{
+	/// Create new headers source using given client.
+	pub fn new(
+		client: SourceClnt,
+		maximal_header_number: Option<RequiredHeaderNumberRef<P::SourceChain>>,
+	) -> Self {
+		BizinikiwiFinalitySource { client, maximal_header_number }
+	}
+
+	/// Returns reference to the underlying RPC client.
+	pub fn client(&self) -> &SourceClnt {
+		&self.client
+	}
+
+	/// Returns best finalized block number.
+	pub async fn on_chain_best_finalized_block_number(
+		&self,
+	) -> Result<BlockNumberOf<P::SourceChain>, Error> {
+		// we **CAN** continue to relay finality proofs if source node is out of sync, because
+		// target node may be missing proofs that are already available at the source
+		self.client.best_finalized_header_number().await
+	}
+
+	/// Return header and its justification of the given block or its descendant that
+	/// has a GRANDPA justification.
+	///
+	/// This method is optimized for cases when `block_number` is close to the best finalized
+	/// chain block.
+	pub async fn prove_block_finality(
+		&self,
+		block_number: BlockNumberOf<P::SourceChain>,
+	) -> Result<
+		(relay_bizinikiwi_client::SyncHeader<HeaderOf<P::SourceChain>>, BizinikiwiFinalityProof<P>),
+		Error,
+	> {
+		// first, subscribe to proofs
+		let next_persistent_proof =
+			self.persistent_proofs_stream(block_number + One::one()).await?.fuse();
+		let next_ephemeral_proof = self.ephemeral_proofs_stream(block_number).await?.fuse();
+
+		// in perfect world we'll need to return justfication for the requested `block_number`
+		let (header, maybe_proof) = self.header_and_finality_proof(block_number).await?;
+		if let Some(proof) = maybe_proof {
+			return Ok((header, proof));
+		}
+
+		// otherwise we don't care which header to return, so let's select first
+		futures::pin_mut!(next_persistent_proof, next_ephemeral_proof);
+		loop {
+			select! {
+				maybe_header_and_proof = next_persistent_proof.next() => match maybe_header_and_proof {
+					Some(header_and_proof) => return header_and_proof,
+					None => continue,
+				},
+				maybe_header_and_proof = next_ephemeral_proof.next() => match maybe_header_and_proof {
+					Some(header_and_proof) => return header_and_proof,
+					None => continue,
+				},
+				complete => return Err(Error::FinalityProofNotFound(block_number.unique_saturated_into()))
+			}
+		}
+	}
+
+	/// Returns stream of headers and their persistent proofs, starting from given block.
+	async fn persistent_proofs_stream(
+		&self,
+		block_number: BlockNumberOf<P::SourceChain>,
+	) -> Result<
+		impl Stream<
+			Item = Result<
+				(
+					relay_bizinikiwi_client::SyncHeader<HeaderOf<P::SourceChain>>,
+					BizinikiwiFinalityProof<P>,
+				),
+				Error,
+			>,
+		>,
+		Error,
+	> {
+		let client = self.client.clone();
+		let best_finalized_block_number = client.best_finalized_header_number().await?;
+		Ok(try_unfold((client, block_number), move |(client, current_block_number)| async move {
+			// if we've passed the `best_finalized_block_number`, we no longer need persistent
+			// justifications
+			if current_block_number > best_finalized_block_number {
+				return Ok(None);
+			}
+
+			let (header, maybe_proof) =
+				header_and_finality_proof::<P>(&client, current_block_number).await?;
+			let next_block_number = current_block_number + One::one();
+			let next_state = (client, next_block_number);
+
+			Ok(Some((maybe_proof.map(|proof| (header, proof)), next_state)))
+		})
+		.try_filter_map(|maybe_result| async { Ok(maybe_result) }))
+	}
+
+	/// Returns stream of headers and their ephemeral proofs, starting from given block.
+	async fn ephemeral_proofs_stream(
+		&self,
+		block_number: BlockNumberOf<P::SourceChain>,
+	) -> Result<
+		impl Stream<
+			Item = Result<
+				(
+					relay_bizinikiwi_client::SyncHeader<HeaderOf<P::SourceChain>>,
+					BizinikiwiFinalityProof<P>,
+				),
+				Error,
+			>,
+		>,
+		Error,
+	> {
+		let client = self.client.clone();
+		Ok(self.finality_proofs().await?.map(Ok).try_filter_map(move |proof| {
+			let client = client.clone();
+			async move {
+				if proof.target_header_number() < block_number {
+					return Ok(None);
+				}
+
+				let header = client.header_by_number(proof.target_header_number()).await?;
+				Ok(Some((header.into(), proof)))
+			}
+		}))
+	}
+}
+
+impl<P: BizinikiwiFinalitySyncPipeline, SourceClnt: Clone> Clone
+	for BizinikiwiFinalitySource<P, SourceClnt>
+{
+	fn clone(&self) -> Self {
+		BizinikiwiFinalitySource {
+			client: self.client.clone(),
+			maximal_header_number: self.maximal_header_number.clone(),
+		}
+	}
+}
+
+#[async_trait]
+impl<P: BizinikiwiFinalitySyncPipeline, SourceClnt: Client<P::SourceChain>> RelayClient
+	for BizinikiwiFinalitySource<P, SourceClnt>
+{
+	type Error = Error;
+
+	async fn reconnect(&mut self) -> Result<(), Error> {
+		self.client.reconnect().await
+	}
+}
+
+#[async_trait]
+impl<P: BizinikiwiFinalitySyncPipeline, SourceClnt: Client<P::SourceChain>>
+	SourceClientBase<FinalitySyncPipelineAdapter<P>> for BizinikiwiFinalitySource<P, SourceClnt>
+{
+	type FinalityProofsStream = BizinikiwiFinalityProofsStream<P>;
+
+	async fn finality_proofs(&self) -> Result<Self::FinalityProofsStream, Error> {
+		finality_proofs::<P>(&self.client).await
+	}
+}
+
+#[async_trait]
+impl<P: BizinikiwiFinalitySyncPipeline, SourceClnt: Client<P::SourceChain>>
+	SourceClient<FinalitySyncPipelineAdapter<P>> for BizinikiwiFinalitySource<P, SourceClnt>
+{
+	async fn best_finalized_block_number(&self) -> Result<BlockNumberOf<P::SourceChain>, Error> {
+		let mut finalized_header_number = self.on_chain_best_finalized_block_number().await?;
+		// never return block number larger than requested. This way we'll never sync headers
+		// past `maximal_header_number`
+		if let Some(ref maximal_header_number) = self.maximal_header_number {
+			let maximal_header_number = *maximal_header_number.lock().await;
+			if finalized_header_number > maximal_header_number {
+				finalized_header_number = maximal_header_number;
+			}
+		}
+		Ok(finalized_header_number)
+	}
+
+	async fn header_and_finality_proof(
+		&self,
+		number: BlockNumberOf<P::SourceChain>,
+	) -> Result<
+		(
+			relay_bizinikiwi_client::SyncHeader<HeaderOf<P::SourceChain>>,
+			Option<BizinikiwiFinalityProof<P>>,
+		),
+		Error,
+	> {
+		header_and_finality_proof::<P>(&self.client, number).await
+	}
+}
+
+async fn header_and_finality_proof<P: BizinikiwiFinalitySyncPipeline>(
+	client: &impl Client<P::SourceChain>,
+	number: BlockNumberOf<P::SourceChain>,
+) -> Result<
+	(
+		relay_bizinikiwi_client::SyncHeader<HeaderOf<P::SourceChain>>,
+		Option<BizinikiwiFinalityProof<P>>,
+	),
+	Error,
+> {
+	let header_hash = client.header_hash_by_number(number).await?;
+	let signed_block = client.block_by_hash(header_hash).await?;
+
+	let justification = signed_block
+		.justification(P::FinalityEngine::ID)
+		.map(|raw_justification| {
+			BizinikiwiFinalityProof::<P>::decode(&mut raw_justification.as_slice())
+		})
+		.transpose()
+		.map_err(Error::ResponseParseFailed)?;
+
+	Ok((signed_block.header().into(), justification))
+}

pezbridges/relays/lib-bizinikiwi-relay/src/finality/target.rs

@@ -0,0 +1,177 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Bizinikiwi client as Bizinikiwi finality proof target.
+
+use crate::{
+	finality::{
+		FinalitySyncPipelineAdapter, SubmitFinalityProofCallBuilder, BizinikiwiFinalitySyncPipeline,
+	},
+	finality_base::{best_synced_header_id, engine::Engine, BizinikiwiFinalityProof},
+	TransactionParams,
+};
+
+use async_trait::async_trait;
+use pezbp_runtime::BlockNumberOf;
+use pez_finality_relay::TargetClient;
+use relay_bizinikiwi_client::{
+	AccountIdOf, AccountKeyPairOf, Chain, Client, Error, HeaderIdOf, HeaderOf, SyncHeader,
+	TransactionEra, TransactionTracker, UnsignedTransaction,
+};
+use relay_utils::relay_loop::Client as RelayClient;
+use pezsp_core::Pair;
+use pezsp_runtime::traits::Header;
+
+/// Bizinikiwi client as Bizinikiwi finality target.
+pub struct BizinikiwiFinalityTarget<P: BizinikiwiFinalitySyncPipeline, TargetClnt> {
+	client: TargetClnt,
+	transaction_params: TransactionParams<AccountKeyPairOf<P::TargetChain>>,
+}
+
+impl<P: BizinikiwiFinalitySyncPipeline, TargetClnt: Client<P::TargetChain>>
+	BizinikiwiFinalityTarget<P, TargetClnt>
+{
+	/// Create new Bizinikiwi headers target.
+	pub fn new(
+		client: TargetClnt,
+		transaction_params: TransactionParams<AccountKeyPairOf<P::TargetChain>>,
+	) -> Self {
+		BizinikiwiFinalityTarget { client, transaction_params }
+	}
+
+	/// Ensure that the bridge pezpallet at target chain is active.
+	pub async fn ensure_pallet_active(&self) -> Result<(), Error> {
+		let is_halted = P::FinalityEngine::is_halted(&self.client).await?;
+		if is_halted {
+			return Err(Error::BridgePalletIsHalted);
+		}
+
+		let is_initialized = P::FinalityEngine::is_initialized(&self.client).await?;
+		if !is_initialized {
+			return Err(Error::BridgePalletIsNotInitialized);
+		}
+
+		Ok(())
+	}
+}
+
+impl<P: BizinikiwiFinalitySyncPipeline, TargetClnt: Clone> Clone
+	for BizinikiwiFinalityTarget<P, TargetClnt>
+{
+	fn clone(&self) -> Self {
+		BizinikiwiFinalityTarget {
+			client: self.client.clone(),
+			transaction_params: self.transaction_params.clone(),
+		}
+	}
+}
+
+#[async_trait]
+impl<P: BizinikiwiFinalitySyncPipeline, TargetClnt: Client<P::TargetChain>> RelayClient
+	for BizinikiwiFinalityTarget<P, TargetClnt>
+{
+	type Error = Error;
+
+	async fn reconnect(&mut self) -> Result<(), Error> {
+		self.client.reconnect().await
+	}
+}
+
+#[async_trait]
+impl<P: BizinikiwiFinalitySyncPipeline, TargetClnt: Client<P::TargetChain>>
+	TargetClient<FinalitySyncPipelineAdapter<P>> for BizinikiwiFinalityTarget<P, TargetClnt>
+where
+	AccountIdOf<P::TargetChain>: From<<AccountKeyPairOf<P::TargetChain> as Pair>::Public>,
+{
+	type TransactionTracker = TransactionTracker<P::TargetChain, TargetClnt>;
+
+	async fn best_finalized_source_block_id(&self) -> Result<HeaderIdOf<P::SourceChain>, Error> {
+		// we can't continue to relay finality if target node is out of sync, because
+		// it may have already received (some of) headers that we're going to relay
+		self.client.ensure_synced().await?;
+		// we can't relay finality if bridge pezpallet at target chain is halted
+		self.ensure_pallet_active().await?;
+
+		Ok(best_synced_header_id::<P::SourceChain, P::TargetChain>(
+			&self.client,
+			self.client.best_header().await?.hash(),
+		)
+		.await?
+		.ok_or(Error::BridgePalletIsNotInitialized)?)
+	}
+
+	async fn free_source_headers_interval(
+		&self,
+	) -> Result<Option<BlockNumberOf<P::SourceChain>>, Self::Error> {
+		Ok(self
+			.client
+			.state_call(
+				self.client.best_header().await?.hash(),
+				P::SourceChain::FREE_HEADERS_INTERVAL_METHOD.into(),
+				(),
+			)
+			.await
+			.unwrap_or_else(|e| {
+				tracing::info!(
+					target: "bridge",
+					error=?e,
+					method=%P::SourceChain::FREE_HEADERS_INTERVAL_METHOD,
+					target=%P::TargetChain::NAME,
+					"Call has failed. Treating as `None`"
+				);
+				None
+			}))
+	}
+
+	async fn submit_finality_proof(
+		&self,
+		header: SyncHeader<HeaderOf<P::SourceChain>>,
+		mut proof: BizinikiwiFinalityProof<P>,
+		is_free_execution_expected: bool,
+	) -> Result<Self::TransactionTracker, Error> {
+		// verify and runtime module at target chain may require optimized finality proof
+		let context =
+			P::FinalityEngine::verify_and_optimize_proof(&self.client, &header, &mut proof).await?;
+
+		// if free execution is expected, but the call size/weight exceeds hardcoded limits, the
+		// runtime may still accept the proof, but it may have some cost for relayer. Let's check
+		// it here to avoid losing relayer funds
+		if is_free_execution_expected {
+			let extras = P::FinalityEngine::check_max_expected_call_limits(&header, &proof);
+			if extras.is_weight_limit_exceeded || extras.extra_size != 0 {
+				return Err(Error::FinalityProofWeightLimitExceeded { extras });
+			}
+		}
+
+		// now we may submit optimized finality proof
+		let mortality = self.transaction_params.mortality;
+		let call = P::SubmitFinalityProofCallBuilder::build_submit_finality_proof_call(
+			header,
+			proof,
+			is_free_execution_expected,
+			context,
+		);
+		self.client
+			.submit_and_watch_signed_extrinsic(
+				&self.transaction_params.signer,
+				move |best_block_id, transaction_nonce| {
+					Ok(UnsignedTransaction::new(call.into(), transaction_nonce)
+						.era(TransactionEra::new(best_block_id, mortality)))
+				},
+			)
+			.await
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/finality_base/engine.rs

@@ -0,0 +1,445 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Support of different finality engines, available in Bizinikiwi.
+
+use crate::error::Error;
+use async_trait::async_trait;
+use bp_header_pez_chain::{
+	justification::{
+		verify_and_optimize_justification, GrandpaEquivocationsFinder, GrandpaJustification,
+		JustificationVerificationContext,
+	},
+	AuthoritySet, ConsensusLogReader, FinalityProof, FindEquivocations, GrandpaConsensusLogReader,
+	HeaderFinalityInfo, HeaderGrandpaInfo, StoredHeaderGrandpaInfo, SubmitFinalityProofCallExtras,
+};
+use pezbp_runtime::{BasicOperatingMode, HeaderIdProvider, OperatingMode};
+use codec::{Decode, Encode};
+use futures::stream::StreamExt;
+use num_traits::{One, Zero};
+use relay_bizinikiwi_client::{
+	BlockNumberOf, Chain, ChainWithGrandpa, Client, Error as BizinikiwiError, HashOf, HeaderOf,
+	Subscription,
+};
+use pezsp_consensus_grandpa::{AuthorityList as GrandpaAuthoritiesSet, GRANDPA_ENGINE_ID};
+use pezsp_core::{storage::StorageKey, Bytes};
+use pezsp_runtime::{scale_info::TypeInfo, traits::Header, ConsensusEngineId};
+use std::{fmt::Debug, marker::PhantomData};
+
+/// Finality engine, used by the Bizinikiwi chain.
+#[async_trait]
+pub trait Engine<C: Chain>: Send {
+	/// Unique consensus engine identifier.
+	const ID: ConsensusEngineId;
+	/// A reader that can extract the consensus log from the header digest and interpret it.
+	type ConsensusLogReader: ConsensusLogReader;
+	/// Type of finality proofs, used by consensus engine.
+	type FinalityProof: FinalityProof<HashOf<C>, BlockNumberOf<C>> + Decode + Encode;
+	/// The context needed for verifying finality proofs.
+	type FinalityVerificationContext: Debug + Send;
+	/// The type of the equivocation proof used by the consensus engine.
+	type EquivocationProof: Clone + Debug + Send + Sync;
+	/// The equivocations finder.
+	type EquivocationsFinder: FindEquivocations<
+		Self::FinalityProof,
+		Self::FinalityVerificationContext,
+		Self::EquivocationProof,
+	>;
+	/// The type of the key owner proof used by the consensus engine.
+	type KeyOwnerProof: Send;
+	/// Type of bridge pezpallet initialization data.
+	type InitializationData: Debug + Send + Sync + 'static;
+	/// Type of bridge pezpallet operating mode.
+	type OperatingMode: OperatingMode + 'static;
+
+	/// Returns storage at the bridged (target) chain that corresponds to some value that is
+	/// missing from the storage until bridge pezpallet is initialized.
+	///
+	/// Note that we don't care about type of the value - just if it present or not.
+	fn is_initialized_key() -> StorageKey;
+
+	/// Returns `Ok(true)` if finality pezpallet at the bridged chain has already been initialized.
+	async fn is_initialized<TargetChain: Chain>(
+		target_client: &impl Client<TargetChain>,
+	) -> Result<bool, BizinikiwiError> {
+		Ok(target_client
+			.raw_storage_value(target_client.best_header_hash().await?, Self::is_initialized_key())
+			.await?
+			.is_some())
+	}
+
+	/// Returns storage key at the bridged (target) chain that corresponds to the variable
+	/// that holds the operating mode of the pezpallet.
+	fn pezpallet_operating_mode_key() -> StorageKey;
+
+	/// Returns `Ok(true)` if finality pezpallet at the bridged chain is halted.
+	async fn is_halted<TargetChain: Chain>(
+		target_client: &impl Client<TargetChain>,
+	) -> Result<bool, BizinikiwiError> {
+		Ok(target_client
+			.storage_value::<Self::OperatingMode>(
+				target_client.best_header_hash().await?,
+				Self::pezpallet_operating_mode_key(),
+			)
+			.await?
+			.map(|operating_mode| operating_mode.is_halted())
+			.unwrap_or(false))
+	}
+
+	/// A method to subscribe to encoded finality proofs, given source client.
+	async fn source_finality_proofs(
+		source_client: &impl Client<C>,
+	) -> Result<Subscription<Bytes>, BizinikiwiError>;
+
+	/// Verify and optimize finality proof before sending it to the target node.
+	///
+	/// Apart from optimization, we expect this method to perform all required checks
+	/// that the `header` and `proof` are valid at the current state of the target chain.
+	async fn verify_and_optimize_proof<TargetChain: Chain>(
+		target_client: &impl Client<TargetChain>,
+		header: &C::Header,
+		proof: &mut Self::FinalityProof,
+	) -> Result<Self::FinalityVerificationContext, BizinikiwiError>;
+
+	/// Checks whether the given `header` and its finality `proof` fit the maximal expected
+	/// call limits (size and weight).
+	fn check_max_expected_call_limits(
+		header: &C::Header,
+		proof: &Self::FinalityProof,
+	) -> SubmitFinalityProofCallExtras;
+
+	/// Prepare initialization data for the finality bridge pezpallet.
+	async fn prepare_initialization_data(
+		client: impl Client<C>,
+	) -> Result<Self::InitializationData, Error<HashOf<C>, BlockNumberOf<C>>>;
+
+	/// Get the context needed for validating a finality proof.
+	async fn finality_verification_context<TargetChain: Chain>(
+		target_client: &impl Client<TargetChain>,
+		at: HashOf<TargetChain>,
+	) -> Result<Self::FinalityVerificationContext, BizinikiwiError>;
+
+	/// Returns the finality info associated to the source headers synced with the target
+	/// at the provided block.
+	async fn synced_headers_finality_info<TargetChain: Chain>(
+		target_client: &impl Client<TargetChain>,
+		at: TargetChain::Hash,
+	) -> Result<
+		Vec<HeaderFinalityInfo<Self::FinalityProof, Self::FinalityVerificationContext>>,
+		BizinikiwiError,
+	>;
+
+	/// Generate key ownership proof for the provided equivocation.
+	async fn generate_source_key_ownership_proof(
+		source_client: &impl Client<C>,
+		at: C::Hash,
+		equivocation: &Self::EquivocationProof,
+	) -> Result<Self::KeyOwnerProof, BizinikiwiError>;
+}
+
+/// GRANDPA finality engine.
+pub struct Grandpa<C>(PhantomData<C>);
+
+impl<C: ChainWithGrandpa> Grandpa<C> {
+	/// Read header by hash from the source client.
+	async fn source_header(
+		source_client: &impl Client<C>,
+		header_hash: C::Hash,
+	) -> Result<C::Header, Error<HashOf<C>, BlockNumberOf<C>>> {
+		source_client
+			.header_by_hash(header_hash)
+			.await
+			.map_err(|err| Error::RetrieveHeader(C::NAME, header_hash, err))
+	}
+
+	/// Read GRANDPA authorities set at given header.
+	async fn source_authorities_set(
+		source_client: &impl Client<C>,
+		header_hash: C::Hash,
+	) -> Result<GrandpaAuthoritiesSet, Error<HashOf<C>, BlockNumberOf<C>>> {
+		const SUB_API_GRANDPA_AUTHORITIES: &str = "GrandpaApi_grandpa_authorities";
+
+		source_client
+			.state_call(header_hash, SUB_API_GRANDPA_AUTHORITIES.to_string(), ())
+			.await
+			.map_err(|err| Error::RetrieveAuthorities(C::NAME, header_hash, err))
+	}
+}
+
+#[async_trait]
+impl<C: ChainWithGrandpa> Engine<C> for Grandpa<C> {
+	const ID: ConsensusEngineId = GRANDPA_ENGINE_ID;
+	type ConsensusLogReader = GrandpaConsensusLogReader<<C::Header as Header>::Number>;
+	type FinalityProof = GrandpaJustification<HeaderOf<C>>;
+	type FinalityVerificationContext = JustificationVerificationContext;
+	type EquivocationProof = pezsp_consensus_grandpa::EquivocationProof<HashOf<C>, BlockNumberOf<C>>;
+	type EquivocationsFinder = GrandpaEquivocationsFinder<C>;
+	type KeyOwnerProof = C::KeyOwnerProof;
+	type InitializationData = bp_header_pez_chain::InitializationData<C::Header>;
+	type OperatingMode = BasicOperatingMode;
+
+	fn is_initialized_key() -> StorageKey {
+		bp_header_pez_chain::storage_keys::best_finalized_key(C::WITH_CHAIN_GRANDPA_PALLET_NAME)
+	}
+
+	fn pezpallet_operating_mode_key() -> StorageKey {
+		bp_header_pez_chain::storage_keys::pezpallet_operating_mode_key(C::WITH_CHAIN_GRANDPA_PALLET_NAME)
+	}
+
+	async fn source_finality_proofs(
+		client: &impl Client<C>,
+	) -> Result<Subscription<Bytes>, BizinikiwiError> {
+		client.subscribe_grandpa_finality_justifications().await
+	}
+
+	async fn verify_and_optimize_proof<TargetChain: Chain>(
+		target_client: &impl Client<TargetChain>,
+		header: &C::Header,
+		proof: &mut Self::FinalityProof,
+	) -> Result<Self::FinalityVerificationContext, BizinikiwiError> {
+		let verification_context = Grandpa::<C>::finality_verification_context(
+			target_client,
+			target_client.best_header().await?.hash(),
+		)
+		.await?;
+		// we're risking with race here - we have decided to submit justification some time ago and
+		// actual authorities set (which we have read now) may have changed, so this
+		// `optimize_justification` may fail. But if target chain is configured properly, it'll fail
+		// anyway, after we submit transaction and failing earlier is better. So - it is fine
+		verify_and_optimize_justification(
+			(header.hash(), *header.number()),
+			&verification_context,
+			proof,
+		)
+		.map(|_| verification_context)
+		.map_err(|e| {
+			BizinikiwiError::Custom(format!(
+				"Failed to optimize {} GRANDPA jutification for header {:?}: {:?}",
+				C::NAME,
+				header.id(),
+				e,
+			))
+		})
+	}
+
+	fn check_max_expected_call_limits(
+		header: &C::Header,
+		proof: &Self::FinalityProof,
+	) -> SubmitFinalityProofCallExtras {
+		bp_header_pez_chain::submit_finality_proof_limits_extras::<C>(header, proof)
+	}
+
+	/// Prepare initialization data for the GRANDPA verifier pezpallet.
+	async fn prepare_initialization_data(
+		source_client: impl Client<C>,
+	) -> Result<Self::InitializationData, Error<HashOf<C>, BlockNumberOf<C>>> {
+		// In ideal world we just need to get best finalized header and then to read GRANDPA
+		// authorities set (`pezpallet_grandpa::CurrentSetId` + `GrandpaApi::grandpa_authorities()`) at
+		// this header.
+		//
+		// But now there are problems with this approach - `CurrentSetId` may return invalid value.
+		// So here we're waiting for the next justification, read the authorities set and then try
+		// to figure out the set id with bruteforce.
+		let mut justifications = Self::source_finality_proofs(&source_client)
+			.await
+			.map_err(|err| Error::Subscribe(C::NAME, err))?;
+		// Read next justification - the header that it finalizes will be used as initial header.
+		let justification = justifications
+			.next()
+			.await
+			.ok_or(Error::ReadJustificationStreamEnded(C::NAME))?;
+
+		// Read initial header.
+		let justification: GrandpaJustification<C::Header> =
+			Decode::decode(&mut &justification.0[..])
+				.map_err(|err| Error::DecodeJustification(C::NAME, err))?;
+
+		let (initial_header_hash, initial_header_number) =
+			(justification.commit.target_hash, justification.commit.target_number);
+
+		let initial_header = Self::source_header(&source_client, initial_header_hash).await?;
+		tracing::trace!(
+			target: "bridge",
+			node=%C::NAME,
+			%initial_header_number,
+			%initial_header_hash,
+			"Selected initial header"
+		);
+
+		// Read GRANDPA authorities set at initial header.
+		let initial_authorities_set =
+			Self::source_authorities_set(&source_client, initial_header_hash).await?;
+		tracing::trace!(
+			target: "bridge",
+			node=%C::NAME,
+			?initial_authorities_set,
+			"Selected"
+		);
+
+		// If initial header changes the GRANDPA authorities set, then we need previous authorities
+		// to verify justification.
+		let mut authorities_for_verification = initial_authorities_set.clone();
+		let scheduled_change = GrandpaConsensusLogReader::<BlockNumberOf<C>>::find_scheduled_change(
+			initial_header.digest(),
+		);
+		assert!(
+			scheduled_change.as_ref().map(|c| c.delay.is_zero()).unwrap_or(true),
+			"GRANDPA authorities change at {} scheduled to happen in {:?} blocks. We expect \
+			regular change to have zero delay",
+			initial_header_hash,
+			scheduled_change.as_ref().map(|c| c.delay),
+		);
+		let schedules_change = scheduled_change.is_some();
+		if schedules_change {
+			authorities_for_verification =
+				Self::source_authorities_set(&source_client, *initial_header.parent_hash()).await?;
+			tracing::trace!(
+				target: "bridge",
+				node=%C::NAME,
+				previous_set=?authorities_for_verification,
+				"Selected header is scheduling GRANDPA authorities set changes."
+			);
+		}
+
+		// Now let's try to guess authorities set id by verifying justification.
+		let mut initial_authorities_set_id = 0;
+		let mut min_possible_block_number = C::BlockNumber::zero();
+		loop {
+			tracing::trace!(
+				target: "bridge",
+				node=%C::NAME,
+				authorities_set_id=%initial_authorities_set_id,
+				"Trying GRANDPA authorities set id"
+			);
+
+			let is_valid_set_id = verify_and_optimize_justification(
+				(initial_header_hash, initial_header_number),
+				&AuthoritySet {
+					authorities: authorities_for_verification.clone(),
+					set_id: initial_authorities_set_id,
+				}
+				.try_into()
+				.map_err(|_| {
+					Error::ReadInvalidAuthorities(C::NAME, authorities_for_verification.clone())
+				})?,
+				&mut justification.clone(),
+			)
+			.is_ok();
+
+			if is_valid_set_id {
+				break;
+			}
+
+			initial_authorities_set_id += 1;
+			min_possible_block_number += One::one();
+			if min_possible_block_number > initial_header_number {
+				// there can't be more authorities set changes than headers => if we have reached
+				// `initial_block_number` and still have not found correct value of
+				// `initial_authorities_set_id`, then something else is broken => fail
+				return Err(Error::GuessInitialAuthorities(C::NAME, initial_header_number));
+			}
+		}
+
+		Ok(bp_header_pez_chain::InitializationData {
+			header: Box::new(initial_header),
+			authority_list: initial_authorities_set,
+			set_id: if schedules_change {
+				initial_authorities_set_id + 1
+			} else {
+				initial_authorities_set_id
+			},
+			operating_mode: BasicOperatingMode::Normal,
+		})
+	}
+
+	async fn finality_verification_context<TargetChain: Chain>(
+		target_client: &impl Client<TargetChain>,
+		at: HashOf<TargetChain>,
+	) -> Result<Self::FinalityVerificationContext, BizinikiwiError> {
+		let current_authority_set_key = bp_header_pez_chain::storage_keys::current_authority_set_key(
+			C::WITH_CHAIN_GRANDPA_PALLET_NAME,
+		);
+		let authority_set: AuthoritySet = target_client
+			.storage_value(at, current_authority_set_key)
+			.await?
+			.map(Ok)
+			.unwrap_or(Err(BizinikiwiError::Custom(format!(
+				"{} `CurrentAuthoritySet` is missing from the {} storage",
+				C::NAME,
+				TargetChain::NAME,
+			))))?;
+
+		authority_set.try_into().map_err(|e| {
+			BizinikiwiError::Custom(format!(
+				"{} `CurrentAuthoritySet` from the {} storage is invalid: {e:?}",
+				C::NAME,
+				TargetChain::NAME,
+			))
+		})
+	}
+
+	async fn synced_headers_finality_info<TargetChain: Chain>(
+		target_client: &impl Client<TargetChain>,
+		at: TargetChain::Hash,
+	) -> Result<Vec<HeaderGrandpaInfo<HeaderOf<C>>>, BizinikiwiError> {
+		let stored_headers_grandpa_info: Vec<StoredHeaderGrandpaInfo<HeaderOf<C>>> = target_client
+			.state_call(at, C::SYNCED_HEADERS_GRANDPA_INFO_METHOD.to_string(), ())
+			.await?;
+
+		let mut headers_grandpa_info = vec![];
+		for stored_header_grandpa_info in stored_headers_grandpa_info {
+			headers_grandpa_info.push(stored_header_grandpa_info.try_into().map_err(|e| {
+				BizinikiwiError::Custom(format!(
+					"{} `AuthoritySet` synced to {} is invalid: {e:?} ",
+					C::NAME,
+					TargetChain::NAME,
+				))
+			})?);
+		}
+
+		Ok(headers_grandpa_info)
+	}
+
+	async fn generate_source_key_ownership_proof(
+		source_client: &impl Client<C>,
+		at: C::Hash,
+		equivocation: &Self::EquivocationProof,
+	) -> Result<Self::KeyOwnerProof, BizinikiwiError> {
+		let set_id = equivocation.set_id();
+		let offender = equivocation.offender();
+
+		let opaque_key_owner_proof = source_client
+			.generate_grandpa_key_ownership_proof(at, set_id, offender.clone())
+			.await?
+			.ok_or(BizinikiwiError::Custom(format!(
+				"Couldn't get GRANDPA key ownership proof from {} at block: {at} \
+				for offender: {:?}, set_id: {set_id} ",
+				C::NAME,
+				offender.clone(),
+			)))?;
+
+		let key_owner_proof =
+			opaque_key_owner_proof.decode().ok_or(BizinikiwiError::Custom(format!(
+				"Couldn't decode GRANDPA `OpaqueKeyOwnnershipProof` from {} at block: {at} 
+				to `{:?}` for offender: {:?}, set_id: {set_id}, at block: {at}",
+				C::NAME,
+				<C::KeyOwnerProof as TypeInfo>::type_info().path,
+				offender.clone(),
+			)))?;
+
+		Ok(key_owner_proof)
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/finality_base/mod.rs

@@ -0,0 +1,106 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Types and functions intended to ease adding of new Bizinikiwi -> Bizinikiwi
+//! finality pipelines.
+
+pub mod engine;
+
+use crate::finality_base::engine::Engine;
+
+use async_trait::async_trait;
+use pezbp_runtime::{HashOf, HeaderIdOf};
+use codec::Decode;
+use futures::{stream::unfold, Stream, StreamExt};
+use relay_bizinikiwi_client::{Chain, Client, Error};
+use std::{fmt::Debug, pin::Pin};
+
+/// Bizinikiwi -> Bizinikiwi finality related pipeline.
+#[async_trait]
+pub trait BizinikiwiFinalityPipeline: 'static + Clone + Debug + Send + Sync {
+	/// Headers of this chain are submitted to the `TargetChain`.
+	type SourceChain: Chain;
+	/// Headers of the `SourceChain` are submitted to this chain.
+	type TargetChain: Chain;
+	/// Finality engine.
+	type FinalityEngine: Engine<Self::SourceChain>;
+}
+
+/// Bizinikiwi finality proof. Specific to the used `FinalityEngine`.
+pub type BizinikiwiFinalityProof<P> = <<P as BizinikiwiFinalityPipeline>::FinalityEngine as Engine<
+	<P as BizinikiwiFinalityPipeline>::SourceChain,
+>>::FinalityProof;
+
+/// Bizinikiwi finality proofs stream.
+pub type BizinikiwiFinalityProofsStream<P> =
+	Pin<Box<dyn Stream<Item = BizinikiwiFinalityProof<P>> + Send>>;
+
+/// Subscribe to new finality proofs.
+pub async fn finality_proofs<P: BizinikiwiFinalityPipeline>(
+	client: &impl Client<P::SourceChain>,
+) -> Result<BizinikiwiFinalityProofsStream<P>, Error> {
+	Ok(unfold(
+		P::FinalityEngine::source_finality_proofs(client).await?,
+		move |mut subscription| async move {
+			loop {
+				let log_error = |err| {
+					tracing::error!(
+						target: "bridge",
+						error=?err,
+						source=%P::SourceChain::NAME,
+						"Failed to read justification target from the justifications stream"
+					);
+				};
+
+				let next_justification = subscription.next().await?;
+
+				let decoded_justification =
+					<P::FinalityEngine as Engine<P::SourceChain>>::FinalityProof::decode(
+						&mut &next_justification[..],
+					);
+
+				let justification = match decoded_justification {
+					Ok(j) => j,
+					Err(err) => {
+						log_error(format!("decode failed with error {err:?}"));
+						continue;
+					},
+				};
+
+				return Some((justification, subscription));
+			}
+		},
+	)
+	.boxed())
+}
+
+/// Get the id of the best `SourceChain` header known to the `TargetChain` at the provided
+/// target block using the exposed runtime API method.
+///
+/// The runtime API method should be `<TargetChain>FinalityApi::best_finalized()`.
+pub async fn best_synced_header_id<SourceChain, TargetChain>(
+	target_client: &impl Client<TargetChain>,
+	at: HashOf<TargetChain>,
+) -> Result<Option<HeaderIdOf<SourceChain>>, Error>
+where
+	SourceChain: Chain,
+	TargetChain: Chain,
+{
+	// now let's read id of best finalized peer header at our best finalized block
+	target_client
+		.state_call(at, SourceChain::BEST_FINALIZED_HEADER_ID_METHOD.into(), ())
+		.await
+}

pezbridges/relays/lib-bizinikiwi-relay/src/lib.rs

@@ -0,0 +1,143 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The library of bizinikiwi relay. contains some public codes to provide to bizinikiwi relay.
+
+#![warn(missing_docs)]
+
+use relay_bizinikiwi_client::{Chain, ChainWithUtilityPallet, UtilityPallet};
+
+use std::marker::PhantomData;
+
+// to avoid `pez_finality_relay` dependency in other crates
+pub use pez_finality_relay::HeadersToRelay;
+
+pub mod cli;
+pub mod equivocation;
+pub mod error;
+pub mod finality;
+pub mod finality_base;
+pub mod messages;
+pub mod on_demand;
+pub mod teyrchains;
+
+/// Transaction creation parameters.
+#[derive(Clone, Debug)]
+pub struct TransactionParams<TS> {
+	/// Transactions author.
+	pub signer: TS,
+	/// Transactions mortality.
+	pub mortality: Option<u32>,
+}
+
+/// Tagged relay account, which balance may be exposed as metrics by the relay.
+#[derive(Clone, Debug)]
+pub enum TaggedAccount<AccountId> {
+	/// Account, used to sign message (also headers and teyrchains) relay transactions from given
+	/// bridged chain.
+	Messages {
+		/// Account id.
+		id: AccountId,
+		/// Name of the bridged chain, which sends us messages or delivery confirmations.
+		bridged_chain: String,
+	},
+}
+
+impl<AccountId> TaggedAccount<AccountId> {
+	/// Returns reference to the account id.
+	pub fn id(&self) -> &AccountId {
+		match *self {
+			TaggedAccount::Messages { ref id, .. } => id,
+		}
+	}
+
+	/// Returns stringified account tag.
+	pub fn tag(&self) -> String {
+		match *self {
+			TaggedAccount::Messages { ref bridged_chain, .. } => {
+				format!("{bridged_chain}Messages")
+			},
+		}
+	}
+}
+
+/// Batch call builder.
+pub trait BatchCallBuilder<Call>: Clone + Send + Sync {
+	/// Create batch call from given calls vector.
+	fn build_batch_call(&self, _calls: Vec<Call>) -> Call;
+}
+
+/// Batch call builder constructor.
+pub trait BatchCallBuilderConstructor<Call>: Clone {
+	/// Call builder, used by this constructor.
+	type CallBuilder: BatchCallBuilder<Call>;
+	/// Create a new instance of a batch call builder.
+	fn new_builder() -> Option<Self::CallBuilder>;
+}
+
+/// Batch call builder based on `pezpallet-utility`.
+#[derive(Clone)]
+pub struct UtilityPalletBatchCallBuilder<C: Chain>(PhantomData<C>);
+
+impl<C: Chain> BatchCallBuilder<C::Call> for UtilityPalletBatchCallBuilder<C>
+where
+	C: ChainWithUtilityPallet,
+{
+	fn build_batch_call(&self, calls: Vec<C::Call>) -> C::Call {
+		C::UtilityPallet::build_batch_call(calls)
+	}
+}
+
+impl<C: Chain> BatchCallBuilderConstructor<C::Call> for UtilityPalletBatchCallBuilder<C>
+where
+	C: ChainWithUtilityPallet,
+{
+	type CallBuilder = Self;
+
+	fn new_builder() -> Option<Self::CallBuilder> {
+		Some(Self(Default::default()))
+	}
+}
+
+// A `BatchCallBuilderConstructor` that always returns `None`.
+impl<Call> BatchCallBuilderConstructor<Call> for () {
+	type CallBuilder = ();
+	fn new_builder() -> Option<Self::CallBuilder> {
+		None
+	}
+}
+
+// Dummy `BatchCallBuilder` implementation that must never be used outside
+// of the `impl BatchCallBuilderConstructor for ()` code.
+impl<Call> BatchCallBuilder<Call> for () {
+	fn build_batch_call(&self, _calls: Vec<Call>) -> Call {
+		unreachable!("never called, because ()::new_builder() returns None; qed")
+	}
+}
+
+/// Module for handling storage proofs compatibility.
+pub mod proofs {
+	use pezbp_runtime::{HashOf, RawStorageProof};
+	use relay_bizinikiwi_client::Chain;
+	use pezsp_trie::StorageProof;
+
+	/// Converts proof to `RawStorageProof` type.
+	pub fn to_raw_storage_proof<SourceChain: Chain>(
+		proof: (StorageProof, HashOf<SourceChain>),
+	) -> RawStorageProof {
+		proof.0.into_iter_nodes().collect()
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/messages/metrics.rs

@@ -0,0 +1,172 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Tools for supporting message lanes between two Bizinikiwi-based chains.
+
+use crate::TaggedAccount;
+
+use bp_relayers::{RewardsAccountOwner, RewardsAccountParams};
+use codec::{Decode, EncodeLike};
+use pezframe_system::AccountInfo;
+use pez_messages_relay::Labeled;
+use pezpallet_balances::AccountData;
+use relay_bizinikiwi_client::{
+	metrics::{FixedU128OrOne, FloatStorageValue, FloatStorageValueMetric},
+	AccountIdOf, BalanceOf, Chain, ChainWithBalances, ChainWithMessages, ChainWithRewards, Client,
+	Error as BizinikiwiError, NonceOf,
+};
+use relay_utils::metrics::{MetricsParams, StandaloneMetric};
+use pezsp_core::storage::StorageData;
+use pezsp_runtime::{FixedPointNumber, FixedU128};
+use std::{fmt::Debug, marker::PhantomData};
+
+/// Add relay accounts balance metrics.
+pub async fn add_relay_balances_metrics<C: ChainWithBalances>(
+	client: impl Client<C>,
+	metrics: &MetricsParams,
+	relay_accounts: &Vec<TaggedAccount<AccountIdOf<C>>>,
+) -> anyhow::Result<()>
+where
+	BalanceOf<C>: Into<u128> + std::fmt::Debug,
+{
+	if relay_accounts.is_empty() {
+		return Ok(());
+	}
+
+	// if `tokenDecimals` is missing from system properties, we'll be using
+	let token_decimals = client
+		.token_decimals()
+		.await?
+		.inspect(|token_decimals| {
+			tracing::info!(target: "bridge", node=%C::NAME, %token_decimals, "Read `tokenDecimals`");
+		})
+		.unwrap_or_else(|| {
+			// turns out it is normal not to have this property - e.g. when pezkuwi binary is
+			// started using `pezkuwi-local` chain. Let's use minimal nominal here
+			tracing::info!(target: "bridge", node=%C::NAME, "Using default (zero) `tokenDecimals`");
+			0
+		});
+	let token_decimals = u32::try_from(token_decimals).map_err(|e| {
+		anyhow::format_err!(
+			"Token decimals value ({}) of {} doesn't fit into u32: {:?}",
+			token_decimals,
+			C::NAME,
+			e,
+		)
+	})?;
+
+	for account in relay_accounts {
+		let relay_account_balance_metric = FloatStorageValueMetric::new(
+			AccountBalanceFromAccountInfo::<C> { token_decimals, _phantom: Default::default() },
+			client.clone(),
+			C::account_info_storage_key(account.id()),
+			format!("at_{}_relay_{}_balance", C::NAME, account.tag()),
+			format!("Balance of the {} relay account at the {}", account.tag(), C::NAME),
+		)?;
+		relay_account_balance_metric.register_and_spawn(&metrics.registry)?;
+	}
+
+	Ok(())
+}
+
+/// Add relay accounts rewards metrics.
+pub async fn add_relay_rewards_metrics<C: ChainWithRewards, BC: ChainWithMessages, LaneId>(
+	client: impl Client<C>,
+	metrics: &MetricsParams,
+	relay_accounts: &Vec<TaggedAccount<AccountIdOf<C>>>,
+	lanes: &[LaneId],
+) -> anyhow::Result<()>
+where
+	C::RewardBalance: Into<u128> + std::fmt::Debug,
+	C::Reward: From<RewardsAccountParams<LaneId>>,
+	LaneId: Clone + Copy + Decode + EncodeLike + Send + Sync + Labeled,
+{
+	if relay_accounts.is_empty() {
+		return Ok(());
+	}
+
+	for account in relay_accounts {
+		if let Some(_) = C::WITH_CHAIN_RELAYERS_PALLET_NAME {
+			for lane in lanes {
+				FloatStorageValueMetric::new(
+					FixedU128OrOne,
+					client.clone(),
+					C::account_reward_storage_key(account.id(), RewardsAccountParams::new(*lane, BC::ID, RewardsAccountOwner::ThisChain)),
+					format!("at_{}_relay_{}_reward_for_msgs_from_{}_on_lane_{}", C::NAME, account.tag(), BC::NAME, lane.label()),
+					format!("Reward of the {} relay account at {} for delivering messages from {} on lane {:?}", account.tag(), C::NAME, BC::NAME, lane.label()),
+				)?.register_and_spawn(&metrics.registry)?;
+
+				FloatStorageValueMetric::new(
+					FixedU128OrOne,
+					client.clone(),
+					C::account_reward_storage_key(account.id(), RewardsAccountParams::new(*lane, BC::ID, RewardsAccountOwner::BridgedChain)),
+					format!("at_{}_relay_{}_reward_for_msgs_to_{}_on_lane_{}", C::NAME, account.tag(), BC::NAME, lane.label()),
+					format!("Reward of the {} relay account at {} for delivering messages confirmations from {} on lane {:?}", account.tag(), C::NAME, BC::NAME, lane.label()),
+				)?.register_and_spawn(&metrics.registry)?;
+			}
+		}
+	}
+
+	Ok(())
+}
+
+/// Adapter for `FloatStorageValueMetric` to decode account free balance.
+#[derive(Clone, Debug)]
+struct AccountBalanceFromAccountInfo<C> {
+	token_decimals: u32,
+	_phantom: PhantomData<C>,
+}
+
+impl<C> FloatStorageValue for AccountBalanceFromAccountInfo<C>
+where
+	C: Chain,
+	BalanceOf<C>: Into<u128>,
+{
+	type Value = FixedU128;
+
+	fn decode(
+		&self,
+		maybe_raw_value: Option<StorageData>,
+	) -> Result<Option<Self::Value>, BizinikiwiError> {
+		maybe_raw_value
+			.map(|raw_value| {
+				AccountInfo::<NonceOf<C>, AccountData<BalanceOf<C>>>::decode(&mut &raw_value.0[..])
+					.map_err(BizinikiwiError::ResponseParseFailed)
+					.map(|account_data| {
+						convert_to_token_balance(account_data.data.free.into(), self.token_decimals)
+					})
+			})
+			.transpose()
+	}
+}
+
+/// Convert from raw `u128` balance (nominated in smallest chain token units) to the float regular
+/// tokens value.
+fn convert_to_token_balance(balance: u128, token_decimals: u32) -> FixedU128 {
+	FixedU128::from_inner(balance.saturating_mul(FixedU128::DIV / 10u128.pow(token_decimals)))
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	#[test]
+	fn token_decimals_used_properly() {
+		let plancks = 425_000_000_000;
+		let token_decimals = 10;
+		let dots = convert_to_token_balance(plancks, token_decimals);
+		assert_eq!(dots, FixedU128::saturating_from_rational(425, 10));
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/messages/source.rs

@@ -0,0 +1,792 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Bizinikiwi client as Bizinikiwi messages source. The chain we connect to should have
+//! runtime that implements `<BridgedChainName>HeaderApi` to allow bridging with
+//! `<BridgedName>` chain.
+
+use crate::{
+	finality_base::best_synced_header_id,
+	messages::{
+		BatchProofTransaction, MessageLaneAdapter, ReceiveMessagesDeliveryProofCallBuilder,
+		BizinikiwiMessageLane,
+	},
+	on_demand::OnDemandRelay,
+	proofs::to_raw_storage_proof,
+	TransactionParams,
+};
+
+use async_std::sync::Arc;
+use async_trait::async_trait;
+use bp_messages::{
+	storage_keys::{operating_mode_key, outbound_lane_data_key},
+	target_chain::FromBridgedChainMessagesProof,
+	ChainWithMessages as _, InboundMessageDetails, MessageNonce, MessagePayload,
+	MessagesOperatingMode, OutboundMessageDetails,
+};
+use pezbp_runtime::{BasicOperatingMode, HeaderIdProvider, RangeInclusiveExt};
+use codec::{Decode, Encode};
+use pezframe_support::weights::Weight;
+use pez_messages_relay::{
+	message_lane::{MessageLane, SourceHeaderIdOf, TargetHeaderIdOf},
+	message_lane_loop::{
+		ClientState, MessageDetails, MessageDetailsMap, MessageProofParameters, SourceClient,
+		SourceClientState,
+	},
+};
+use num_traits::Zero;
+use relay_bizinikiwi_client::{
+	AccountIdOf, AccountKeyPairOf, BalanceOf, Chain, ChainWithMessages, Client,
+	Error as BizinikiwiError, HashOf, HeaderIdOf, TransactionEra, TransactionTracker,
+	UnsignedTransaction,
+};
+use relay_utils::relay_loop::Client as RelayClient;
+use pezsp_core::Pair;
+use std::ops::RangeInclusive;
+
+/// Intermediate message proof returned by the source Bizinikiwi node. Includes everything
+/// required to submit to the target node: cumulative dispatch weight of bundled messages and
+/// the proof itself.
+pub type BizinikiwiMessagesProof<C, L> = (Weight, FromBridgedChainMessagesProof<HashOf<C>, L>);
+type MessagesToRefine<'a> = Vec<(MessagePayload, &'a mut OutboundMessageDetails)>;
+
+/// Outbound lane data - for backwards compatibility with `bp_messages::OutboundLaneData` which has
+/// additional `lane_state` attribute.
+///
+/// TODO: remove - https://github.com/pezkuwichain/pezkuwi-sdk/issues/22
+#[derive(Decode)]
+struct LegacyOutboundLaneData {
+	#[allow(unused)]
+	oldest_unpruned_nonce: MessageNonce,
+	latest_received_nonce: MessageNonce,
+	latest_generated_nonce: MessageNonce,
+}
+
+/// Bizinikiwi client as Bizinikiwi messages source.
+pub struct BizinikiwiMessagesSource<P: BizinikiwiMessageLane, SourceClnt, TargetClnt> {
+	source_client: SourceClnt,
+	target_client: TargetClnt,
+	lane_id: P::LaneId,
+	transaction_params: TransactionParams<AccountKeyPairOf<P::SourceChain>>,
+	target_to_source_headers_relay: Option<Arc<dyn OnDemandRelay<P::TargetChain, P::SourceChain>>>,
+}
+
+impl<P: BizinikiwiMessageLane, SourceClnt: Client<P::SourceChain>, TargetClnt>
+	BizinikiwiMessagesSource<P, SourceClnt, TargetClnt>
+{
+	/// Create new Bizinikiwi headers source.
+	pub fn new(
+		source_client: SourceClnt,
+		target_client: TargetClnt,
+		lane_id: P::LaneId,
+		transaction_params: TransactionParams<AccountKeyPairOf<P::SourceChain>>,
+		target_to_source_headers_relay: Option<
+			Arc<dyn OnDemandRelay<P::TargetChain, P::SourceChain>>,
+		>,
+	) -> Self {
+		BizinikiwiMessagesSource {
+			source_client,
+			target_client,
+			lane_id,
+			transaction_params,
+			target_to_source_headers_relay,
+		}
+	}
+
+	/// Read outbound lane state from the on-chain storage at given block.
+	async fn outbound_lane_data(
+		&self,
+		id: SourceHeaderIdOf<MessageLaneAdapter<P>>,
+	) -> Result<Option<LegacyOutboundLaneData>, BizinikiwiError> {
+		self.source_client
+			.storage_value(
+				id.hash(),
+				outbound_lane_data_key(
+					P::TargetChain::WITH_CHAIN_MESSAGES_PALLET_NAME,
+					&self.lane_id,
+				),
+			)
+			.await
+	}
+
+	/// Ensure that the messages pezpallet at source chain is active.
+	async fn ensure_pallet_active(&self) -> Result<(), BizinikiwiError> {
+		ensure_messages_pallet_active::<P::SourceChain, P::TargetChain, _>(&self.source_client)
+			.await
+	}
+}
+
+impl<P: BizinikiwiMessageLane, SourceClnt: Clone, TargetClnt: Clone> Clone
+	for BizinikiwiMessagesSource<P, SourceClnt, TargetClnt>
+{
+	fn clone(&self) -> Self {
+		Self {
+			source_client: self.source_client.clone(),
+			target_client: self.target_client.clone(),
+			lane_id: self.lane_id,
+			transaction_params: self.transaction_params.clone(),
+			target_to_source_headers_relay: self.target_to_source_headers_relay.clone(),
+		}
+	}
+}
+
+#[async_trait]
+impl<
+		P: BizinikiwiMessageLane,
+		SourceClnt: Client<P::SourceChain>,
+		TargetClnt: Client<P::TargetChain>,
+	> RelayClient for BizinikiwiMessagesSource<P, SourceClnt, TargetClnt>
+{
+	type Error = BizinikiwiError;
+
+	async fn reconnect(&mut self) -> Result<(), BizinikiwiError> {
+		// since the client calls RPC methods on both sides, we need to reconnect both
+		self.source_client.reconnect().await?;
+		self.target_client.reconnect().await?;
+
+		// call reconnect on on-demand headers relay, because we may use different chains there
+		// and the error that has lead to reconnect may have came from those other chains
+		// (see `require_target_header_on_source`)
+		//
+		// this may lead to multiple reconnects to the same node during the same call and it
+		// needs to be addressed in the future
+		// TODO: https://github.com/pezkuwichain/kurdistan-sdk/issues/82
+		if let Some(ref mut target_to_source_headers_relay) = self.target_to_source_headers_relay {
+			target_to_source_headers_relay.reconnect().await?;
+		}
+
+		Ok(())
+	}
+}
+
+#[async_trait]
+impl<
+		P: BizinikiwiMessageLane,
+		SourceClnt: Client<P::SourceChain>,
+		TargetClnt: Client<P::TargetChain>,
+	> SourceClient<MessageLaneAdapter<P>> for BizinikiwiMessagesSource<P, SourceClnt, TargetClnt>
+where
+	AccountIdOf<P::SourceChain>: From<<AccountKeyPairOf<P::SourceChain> as Pair>::Public>,
+{
+	type BatchTransaction =
+		BatchProofTransaction<P::SourceChain, P::TargetChain, P::SourceBatchCallBuilder>;
+	type TransactionTracker = TransactionTracker<P::SourceChain, SourceClnt>;
+
+	async fn state(&self) -> Result<SourceClientState<MessageLaneAdapter<P>>, BizinikiwiError> {
+		// we can't continue to deliver confirmations if source node is out of sync, because
+		// it may have already received confirmations that we're going to deliver
+		//
+		// we can't continue to deliver messages if target node is out of sync, because
+		// it may have already received (some of) messages that we're going to deliver
+		self.source_client.ensure_synced().await?;
+		self.target_client.ensure_synced().await?;
+		// we can't relay confirmations if messages pezpallet at source chain is halted
+		self.ensure_pallet_active().await?;
+
+		read_client_state_from_both_chains(&self.source_client, &self.target_client).await
+	}
+
+	async fn latest_generated_nonce(
+		&self,
+		id: SourceHeaderIdOf<MessageLaneAdapter<P>>,
+	) -> Result<(SourceHeaderIdOf<MessageLaneAdapter<P>>, MessageNonce), BizinikiwiError> {
+		// lane data missing from the storage is fine until first message is sent
+		let latest_generated_nonce = self
+			.outbound_lane_data(id)
+			.await?
+			.map(|data| data.latest_generated_nonce)
+			.unwrap_or(0);
+		Ok((id, latest_generated_nonce))
+	}
+
+	async fn latest_confirmed_received_nonce(
+		&self,
+		id: SourceHeaderIdOf<MessageLaneAdapter<P>>,
+	) -> Result<(SourceHeaderIdOf<MessageLaneAdapter<P>>, MessageNonce), BizinikiwiError> {
+		// lane data missing from the storage is fine until first message is sent
+		let latest_received_nonce = self
+			.outbound_lane_data(id)
+			.await?
+			.map(|data| data.latest_received_nonce)
+			.unwrap_or(0);
+		Ok((id, latest_received_nonce))
+	}
+
+	async fn generated_message_details(
+		&self,
+		id: SourceHeaderIdOf<MessageLaneAdapter<P>>,
+		nonces: RangeInclusive<MessageNonce>,
+	) -> Result<MessageDetailsMap<BalanceOf<P::SourceChain>>, BizinikiwiError> {
+		let mut out_msgs_details: Vec<_> = self
+			.source_client
+			.state_call::<_, Vec<_>>(
+				id.hash(),
+				P::TargetChain::TO_CHAIN_MESSAGE_DETAILS_METHOD.into(),
+				(self.lane_id, *nonces.start(), *nonces.end()),
+			)
+			.await?;
+		validate_out_msgs_details::<P::SourceChain>(&out_msgs_details, nonces)?;
+
+		// prepare arguments of the inbound message details call (if we need it)
+		let mut msgs_to_refine = vec![];
+		for out_msg_details in out_msgs_details.iter_mut() {
+			// in our current strategy all messages are supposed to be paid at the target chain
+
+			// for pay-at-target messages we may want to ask target chain for
+			// refined dispatch weight
+			let msg_key = bp_messages::storage_keys::message_key(
+				P::TargetChain::WITH_CHAIN_MESSAGES_PALLET_NAME,
+				&self.lane_id,
+				out_msg_details.nonce,
+			);
+			let msg_payload: MessagePayload =
+				self.source_client.storage_value(id.hash(), msg_key).await?.ok_or_else(|| {
+					BizinikiwiError::Custom(format!(
+						"Message to {} {:?}/{} is missing from runtime the storage of {} at {:?}",
+						P::TargetChain::NAME,
+						self.lane_id,
+						out_msg_details.nonce,
+						P::SourceChain::NAME,
+						id,
+					))
+				})?;
+
+			msgs_to_refine.push((msg_payload, out_msg_details));
+		}
+
+		let best_target_header_hash = self.target_client.best_header_hash().await?;
+		for mut msgs_to_refine_batch in split_msgs_to_refine::<
+			P::SourceChain,
+			P::TargetChain,
+			P::LaneId,
+		>(self.lane_id, msgs_to_refine)?
+		{
+			let in_msgs_details = self
+				.target_client
+				.state_call::<_, Vec<InboundMessageDetails>>(
+					best_target_header_hash,
+					P::SourceChain::FROM_CHAIN_MESSAGE_DETAILS_METHOD.into(),
+					(self.lane_id, &msgs_to_refine_batch),
+				)
+				.await?;
+			if in_msgs_details.len() != msgs_to_refine_batch.len() {
+				return Err(BizinikiwiError::Custom(format!(
+					"Call of {} at {} has returned {} entries instead of expected {}",
+					P::SourceChain::FROM_CHAIN_MESSAGE_DETAILS_METHOD,
+					P::TargetChain::NAME,
+					in_msgs_details.len(),
+					msgs_to_refine_batch.len(),
+				)));
+			}
+			for ((_, out_msg_details), in_msg_details) in
+				msgs_to_refine_batch.iter_mut().zip(in_msgs_details)
+			{
+				tracing::trace!(
+					target: "bridge",
+					source=%P::SourceChain::NAME,
+					target=%P::TargetChain::NAME,
+					lane_id=?self.lane_id,
+					nonce=%out_msg_details.nonce,
+					at_source=%out_msg_details.dispatch_weight,
+					at_target=%in_msg_details.dispatch_weight,
+					"Refined weight of source->target message"
+				);
+				out_msg_details.dispatch_weight = in_msg_details.dispatch_weight;
+			}
+		}
+
+		let mut msgs_details_map = MessageDetailsMap::new();
+		for out_msg_details in out_msgs_details {
+			msgs_details_map.insert(
+				out_msg_details.nonce,
+				MessageDetails {
+					dispatch_weight: out_msg_details.dispatch_weight,
+					size: out_msg_details.size as _,
+					reward: Zero::zero(),
+				},
+			);
+		}
+
+		Ok(msgs_details_map)
+	}
+
+	async fn prove_messages(
+		&self,
+		id: SourceHeaderIdOf<MessageLaneAdapter<P>>,
+		nonces: RangeInclusive<MessageNonce>,
+		proof_parameters: MessageProofParameters,
+	) -> Result<
+		(
+			SourceHeaderIdOf<MessageLaneAdapter<P>>,
+			RangeInclusive<MessageNonce>,
+			<MessageLaneAdapter<P> as MessageLane>::MessagesProof,
+		),
+		BizinikiwiError,
+	> {
+		let mut storage_keys = Vec::with_capacity(nonces.saturating_len() as usize);
+		for message_nonce in nonces.clone() {
+			let message_key = bp_messages::storage_keys::message_key(
+				P::TargetChain::WITH_CHAIN_MESSAGES_PALLET_NAME,
+				&self.lane_id,
+				message_nonce,
+			);
+			storage_keys.push(message_key);
+		}
+		if proof_parameters.outbound_state_proof_required {
+			storage_keys.push(outbound_lane_data_key(
+				P::TargetChain::WITH_CHAIN_MESSAGES_PALLET_NAME,
+				&self.lane_id,
+			));
+		}
+
+		let storage_proof =
+			self.source_client.prove_storage(id.hash(), storage_keys.clone()).await?;
+		let proof = FromBridgedChainMessagesProof {
+			bridged_header_hash: id.1,
+			storage_proof: to_raw_storage_proof::<P::SourceChain>(storage_proof),
+			lane: self.lane_id,
+			nonces_start: *nonces.start(),
+			nonces_end: *nonces.end(),
+		};
+		Ok((id, nonces, (proof_parameters.dispatch_weight, proof)))
+	}
+
+	async fn submit_messages_receiving_proof(
+		&self,
+		maybe_batch_tx: Option<Self::BatchTransaction>,
+		_generated_at_block: TargetHeaderIdOf<MessageLaneAdapter<P>>,
+		proof: <MessageLaneAdapter<P> as MessageLane>::MessagesReceivingProof,
+	) -> Result<Self::TransactionTracker, BizinikiwiError> {
+		let messages_proof_call =
+			P::ReceiveMessagesDeliveryProofCallBuilder::build_receive_messages_delivery_proof_call(
+				proof,
+				maybe_batch_tx.is_none(),
+			);
+		let final_call = match maybe_batch_tx {
+			Some(batch_tx) => batch_tx.append_call_and_build(messages_proof_call),
+			None => messages_proof_call,
+		};
+
+		let transaction_params = self.transaction_params.clone();
+		self.source_client
+			.submit_and_watch_signed_extrinsic(
+				&self.transaction_params.signer,
+				move |best_block_id, transaction_nonce| {
+					Ok(UnsignedTransaction::new(final_call.into(), transaction_nonce)
+						.era(TransactionEra::new(best_block_id, transaction_params.mortality)))
+				},
+			)
+			.await
+	}
+
+	async fn require_target_header_on_source(
+		&self,
+		id: TargetHeaderIdOf<MessageLaneAdapter<P>>,
+	) -> Result<Option<Self::BatchTransaction>, BizinikiwiError> {
+		if let Some(ref target_to_source_headers_relay) = self.target_to_source_headers_relay {
+			if let Some(batch_tx) =
+				BatchProofTransaction::new(target_to_source_headers_relay.clone(), id.0).await?
+			{
+				return Ok(Some(batch_tx));
+			}
+
+			target_to_source_headers_relay.require_more_headers(id.0).await;
+		}
+
+		Ok(None)
+	}
+}
+
+/// Ensure that the messages pezpallet at source chain is active.
+pub(crate) async fn ensure_messages_pallet_active<AtChain, WithChain, AtChainClient>(
+	client: &AtChainClient,
+) -> Result<(), BizinikiwiError>
+where
+	AtChain: ChainWithMessages,
+	WithChain: ChainWithMessages,
+	AtChainClient: Client<AtChain>,
+{
+	let operating_mode = client
+		.storage_value(
+			client.best_header_hash().await?,
+			operating_mode_key(WithChain::WITH_CHAIN_MESSAGES_PALLET_NAME),
+		)
+		.await?;
+	let is_halted =
+		operating_mode == Some(MessagesOperatingMode::Basic(BasicOperatingMode::Halted));
+	if is_halted {
+		Err(BizinikiwiError::BridgePalletIsHalted)
+	} else {
+		Ok(())
+	}
+}
+
+/// Read best blocks from given client.
+///
+/// This function assumes that the chain that is followed by the `self_client` has
+/// bridge GRANDPA pezpallet deployed and it provides `best_finalized_header_id_method_name`
+/// runtime API to read the best finalized Bridged chain header.
+///
+/// The value of `actual_best_finalized_peer_at_best_self` will always match
+/// the `best_finalized_peer_at_best_self`.
+pub async fn read_client_state<SelfChain, PeerChain>(
+	self_client: &impl Client<SelfChain>,
+) -> Result<ClientState<HeaderIdOf<SelfChain>, HeaderIdOf<PeerChain>>, BizinikiwiError>
+where
+	SelfChain: Chain,
+	PeerChain: Chain,
+{
+	// let's read our state first: we need best finalized header hash on **this** chain
+	let self_best_finalized_id = self_client.best_finalized_header().await?.id();
+	// now let's read our best header on **this** chain
+	let self_best_id = self_client.best_header().await?.id();
+
+	// now let's read id of best finalized peer header at our best finalized block
+	let peer_on_self_best_finalized_id =
+		best_synced_header_id::<PeerChain, SelfChain>(self_client, self_best_id.hash()).await?;
+
+	Ok(ClientState {
+		best_self: self_best_id,
+		best_finalized_self: self_best_finalized_id,
+		best_finalized_peer_at_best_self: peer_on_self_best_finalized_id,
+		actual_best_finalized_peer_at_best_self: peer_on_self_best_finalized_id,
+	})
+}
+
+/// Does the same stuff as `read_client_state`, but properly fills the
+/// `actual_best_finalized_peer_at_best_self` field of the result.
+pub async fn read_client_state_from_both_chains<SelfChain, PeerChain>(
+	self_client: &impl Client<SelfChain>,
+	peer_client: &impl Client<PeerChain>,
+) -> Result<ClientState<HeaderIdOf<SelfChain>, HeaderIdOf<PeerChain>>, BizinikiwiError>
+where
+	SelfChain: Chain,
+	PeerChain: Chain,
+{
+	let mut client_state = read_client_state::<SelfChain, PeerChain>(self_client).await?;
+	client_state.actual_best_finalized_peer_at_best_self =
+		match client_state.best_finalized_peer_at_best_self.as_ref() {
+			Some(peer_on_self_best_finalized_id) => {
+				let actual_peer_on_self_best_finalized =
+					peer_client.header_by_number(peer_on_self_best_finalized_id.number()).await?;
+				Some(actual_peer_on_self_best_finalized.id())
+			},
+			_ => client_state.best_finalized_peer_at_best_self,
+		};
+	Ok(client_state)
+}
+
+/// Reads best `PeerChain` header known to the `SelfChain` using provided runtime API method.
+///
+/// Method is supposed to be the `<PeerChain>FinalityApi::best_finalized()` method.
+pub async fn best_finalized_peer_header_at_self<SelfChain, PeerChain>(
+	self_client: &impl Client<SelfChain>,
+	at_self_hash: HashOf<SelfChain>,
+) -> Result<Option<HeaderIdOf<PeerChain>>, BizinikiwiError>
+where
+	SelfChain: Chain,
+	PeerChain: Chain,
+{
+	// now let's read id of best finalized peer header at our best finalized block
+	self_client
+		.state_call::<_, Option<_>>(
+			at_self_hash,
+			PeerChain::BEST_FINALIZED_HEADER_ID_METHOD.into(),
+			(),
+		)
+		.await
+}
+
+fn validate_out_msgs_details<C: Chain>(
+	out_msgs_details: &[OutboundMessageDetails],
+	nonces: RangeInclusive<MessageNonce>,
+) -> Result<(), BizinikiwiError> {
+	let make_missing_nonce_error = |expected_nonce| {
+		Err(BizinikiwiError::Custom(format!(
+			"Missing nonce {expected_nonce} in message_details call result. Expected all nonces from {nonces:?}",
+		)))
+	};
+
+	if out_msgs_details.len() > nonces.clone().count() {
+		return Err(BizinikiwiError::Custom(
+			"More messages than requested returned by the message_details call.".into(),
+		));
+	}
+
+	// Check if last nonce is missing. The loop below is not checking this.
+	if out_msgs_details.is_empty() && !nonces.is_empty() {
+		return make_missing_nonce_error(*nonces.end());
+	}
+
+	let mut nonces_iter = nonces.clone().rev().peekable();
+	let mut out_msgs_details_iter = out_msgs_details.iter().rev();
+	while let Some((out_msg_details, &nonce)) = out_msgs_details_iter.next().zip(nonces_iter.peek())
+	{
+		nonces_iter.next();
+		if out_msg_details.nonce != nonce {
+			// Some nonces are missing from the middle/tail of the range. This is critical error.
+			return make_missing_nonce_error(nonce);
+		}
+	}
+
+	// Check if some nonces from the beginning of the range are missing. This may happen if
+	// some messages were already pruned from the source node. This is not a critical error
+	// and will be auto-resolved by messages lane (and target node).
+	if nonces_iter.peek().is_some() {
+		tracing::info!(
+			target: "bridge",
+			node=%C::NAME,
+			missing=?nonces_iter.rev().collect::<Vec<_>>(),
+			"Some messages are missing. Target node may be out of sync?"
+		);
+	}
+
+	Ok(())
+}
+
+fn split_msgs_to_refine<Source: Chain + ChainWithMessages, Target: Chain, LaneId: Encode + Copy>(
+	lane_id: LaneId,
+	msgs_to_refine: MessagesToRefine,
+) -> Result<Vec<MessagesToRefine>, BizinikiwiError> {
+	let max_batch_size = Target::max_extrinsic_size() as usize;
+	let mut batches = vec![];
+
+	let mut current_msgs_batch = msgs_to_refine;
+	while !current_msgs_batch.is_empty() {
+		let mut next_msgs_batch = vec![];
+		while (lane_id, &current_msgs_batch).encoded_size() > max_batch_size {
+			if current_msgs_batch.len() <= 1 {
+				return Err(BizinikiwiError::Custom(format!(
+					"Call of {} at {} can't be executed even if only one message is supplied. \
+						max_extrinsic_size(): {}",
+					Source::FROM_CHAIN_MESSAGE_DETAILS_METHOD,
+					Target::NAME,
+					Target::max_extrinsic_size(),
+				)));
+			}
+
+			if let Some(msg) = current_msgs_batch.pop() {
+				next_msgs_batch.insert(0, msg);
+			}
+		}
+
+		batches.push(current_msgs_batch);
+		current_msgs_batch = next_msgs_batch;
+	}
+
+	Ok(batches)
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use bp_messages::{HashedLaneId, LaneIdType};
+	use relay_bizinikiwi_client::test_chain::TestChain;
+
+	/// Lane identifier type used for tests.
+	type TestLaneIdType = HashedLaneId;
+
+	fn message_details_from_rpc(
+		nonces: RangeInclusive<MessageNonce>,
+	) -> Vec<OutboundMessageDetails> {
+		nonces
+			.into_iter()
+			.map(|nonce| bp_messages::OutboundMessageDetails {
+				nonce,
+				dispatch_weight: Weight::zero(),
+				size: 0,
+			})
+			.collect()
+	}
+
+	#[test]
+	fn validate_out_msgs_details_succeeds_if_no_messages_are_missing() {
+		assert!(validate_out_msgs_details::<TestChain>(&message_details_from_rpc(1..=3), 1..=3,)
+			.is_ok());
+	}
+
+	#[test]
+	fn validate_out_msgs_details_succeeds_if_head_messages_are_missing() {
+		assert!(validate_out_msgs_details::<TestChain>(&message_details_from_rpc(2..=3), 1..=3,)
+			.is_ok())
+	}
+
+	#[test]
+	fn validate_out_msgs_details_fails_if_mid_messages_are_missing() {
+		let mut message_details_from_rpc = message_details_from_rpc(1..=3);
+		message_details_from_rpc.remove(1);
+		assert!(matches!(
+			validate_out_msgs_details::<TestChain>(&message_details_from_rpc, 1..=3,),
+			Err(BizinikiwiError::Custom(_))
+		));
+	}
+
+	#[test]
+	fn validate_out_msgs_details_map_fails_if_tail_messages_are_missing() {
+		assert!(matches!(
+			validate_out_msgs_details::<TestChain>(&message_details_from_rpc(1..=2), 1..=3,),
+			Err(BizinikiwiError::Custom(_))
+		));
+	}
+
+	#[test]
+	fn validate_out_msgs_details_fails_if_all_messages_are_missing() {
+		assert!(matches!(
+			validate_out_msgs_details::<TestChain>(&[], 1..=3),
+			Err(BizinikiwiError::Custom(_))
+		));
+	}
+
+	#[test]
+	fn validate_out_msgs_details_fails_if_more_messages_than_nonces() {
+		assert!(matches!(
+			validate_out_msgs_details::<TestChain>(&message_details_from_rpc(1..=5), 2..=5,),
+			Err(BizinikiwiError::Custom(_))
+		));
+	}
+
+	fn check_split_msgs_to_refine(
+		payload_sizes: Vec<usize>,
+		expected_batches: Result<Vec<usize>, ()>,
+	) {
+		let mut out_msgs_details = vec![];
+		for (idx, _) in payload_sizes.iter().enumerate() {
+			out_msgs_details.push(OutboundMessageDetails {
+				nonce: idx as MessageNonce,
+				dispatch_weight: Weight::zero(),
+				size: 0,
+			});
+		}
+
+		let mut msgs_to_refine = vec![];
+		for (&payload_size, out_msg_details) in
+			payload_sizes.iter().zip(out_msgs_details.iter_mut())
+		{
+			let payload = vec![1u8; payload_size];
+			msgs_to_refine.push((payload, out_msg_details));
+		}
+
+		let maybe_batches = split_msgs_to_refine::<TestChain, TestChain, TestLaneIdType>(
+			TestLaneIdType::try_new(1, 2).unwrap(),
+			msgs_to_refine,
+		);
+		match expected_batches {
+			Ok(expected_batches) => {
+				let batches = maybe_batches.unwrap();
+				let mut idx = 0;
+				assert_eq!(batches.len(), expected_batches.len());
+				for (batch, &expected_batch_size) in batches.iter().zip(expected_batches.iter()) {
+					assert_eq!(batch.len(), expected_batch_size);
+					for msg_to_refine in batch {
+						assert_eq!(msg_to_refine.0.len(), payload_sizes[idx]);
+						idx += 1;
+					}
+				}
+			},
+			Err(_) => {
+				matches!(maybe_batches, Err(BizinikiwiError::Custom(_)));
+			},
+		}
+	}
+
+	#[test]
+	fn test_split_msgs_to_refine() {
+		let max_extrinsic_size = 100000;
+
+		// Check that an error is returned when one of the messages is too big.
+		check_split_msgs_to_refine(vec![max_extrinsic_size], Err(()));
+		check_split_msgs_to_refine(vec![50, 100, max_extrinsic_size, 200], Err(()));
+
+		// Otherwise check that the split is valid.
+		check_split_msgs_to_refine(vec![100, 200, 300, 400], Ok(vec![4]));
+		check_split_msgs_to_refine(
+			vec![
+				50,
+				100,
+				max_extrinsic_size - 500,
+				500,
+				1000,
+				1500,
+				max_extrinsic_size - 3500,
+				5000,
+				10000,
+			],
+			Ok(vec![3, 4, 2]),
+		);
+		check_split_msgs_to_refine(
+			vec![
+				50,
+				100,
+				max_extrinsic_size - 150,
+				500,
+				1000,
+				1500,
+				max_extrinsic_size - 3000,
+				5000,
+				10000,
+			],
+			Ok(vec![2, 1, 3, 1, 2]),
+		);
+		check_split_msgs_to_refine(
+			vec![
+				5000,
+				10000,
+				max_extrinsic_size - 3500,
+				500,
+				1000,
+				1500,
+				max_extrinsic_size - 500,
+				50,
+				100,
+			],
+			Ok(vec![2, 4, 3]),
+		);
+	}
+
+	#[test]
+	fn outbound_lane_data_wrapper_is_compatible() {
+		let bytes_without_state =
+			vec![1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0];
+		let bytes_with_state = {
+			// add state byte `bp_messages::LaneState::Opened`
+			let mut b = bytes_without_state.clone();
+			b.push(0);
+			b
+		};
+
+		let full = bp_messages::OutboundLaneData {
+			oldest_unpruned_nonce: 1,
+			latest_received_nonce: 2,
+			latest_generated_nonce: 3,
+			state: bp_messages::LaneState::Opened,
+		};
+		assert_eq!(full.encode(), bytes_with_state);
+		assert_ne!(full.encode(), bytes_without_state);
+
+		// decode from `bytes_with_state`
+		let decoded: LegacyOutboundLaneData = Decode::decode(&mut &bytes_with_state[..]).unwrap();
+		assert_eq!(full.oldest_unpruned_nonce, decoded.oldest_unpruned_nonce);
+		assert_eq!(full.latest_received_nonce, decoded.latest_received_nonce);
+		assert_eq!(full.latest_generated_nonce, decoded.latest_generated_nonce);
+
+		// decode from `bytes_without_state`
+		let decoded: LegacyOutboundLaneData =
+			Decode::decode(&mut &bytes_without_state[..]).unwrap();
+		assert_eq!(full.oldest_unpruned_nonce, decoded.oldest_unpruned_nonce);
+		assert_eq!(full.latest_received_nonce, decoded.latest_received_nonce);
+		assert_eq!(full.latest_generated_nonce, decoded.latest_generated_nonce);
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/messages/target.rs

@@ -0,0 +1,410 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Bizinikiwi client as Bizinikiwi messages target. The chain we connect to should have
+//! runtime that implements `<BridgedChainName>HeaderApi` to allow bridging with
+//! `<BridgedName>` chain.
+
+use crate::{
+	messages::{
+		source::{
+			ensure_messages_pallet_active, read_client_state_from_both_chains,
+			BizinikiwiMessagesProof,
+		},
+		BatchProofTransaction, MessageLaneAdapter, ReceiveMessagesProofCallBuilder,
+		BizinikiwiMessageLane,
+	},
+	on_demand::OnDemandRelay,
+	proofs::to_raw_storage_proof,
+	TransactionParams,
+};
+
+use async_std::sync::Arc;
+use async_trait::async_trait;
+use bp_messages::{
+	source_chain::FromBridgedChainMessagesDeliveryProof, storage_keys::inbound_lane_data_key,
+	ChainWithMessages as _, LaneState, MessageNonce, UnrewardedRelayer, UnrewardedRelayersState,
+};
+use codec::Decode;
+use pez_messages_relay::{
+	message_lane::{MessageLane, SourceHeaderIdOf, TargetHeaderIdOf},
+	message_lane_loop::{NoncesSubmitArtifacts, TargetClient, TargetClientState},
+};
+use relay_bizinikiwi_client::{
+	AccountIdOf, AccountKeyPairOf, BalanceOf, CallOf, Chain, Client, Error as BizinikiwiError,
+	HashOf, TransactionEra, TransactionTracker, UnsignedTransaction,
+};
+use relay_utils::relay_loop::Client as RelayClient;
+use pezsp_core::Pair;
+use std::{collections::VecDeque, convert::TryFrom, ops::RangeInclusive};
+
+/// Message receiving proof returned by the target Bizinikiwi node.
+pub type BizinikiwiMessagesDeliveryProof<C, L> =
+	(UnrewardedRelayersState, FromBridgedChainMessagesDeliveryProof<HashOf<C>, L>);
+
+/// Inbound lane data - for backwards compatibility with `bp_messages::InboundLaneData` which has
+/// additional `lane_state` attribute.
+///
+/// TODO: remove - https://github.com/pezkuwichain/pezkuwi-sdk/issues/22
+#[derive(Decode)]
+struct LegacyInboundLaneData<RelayerId> {
+	relayers: VecDeque<UnrewardedRelayer<RelayerId>>,
+	last_confirmed_nonce: MessageNonce,
+}
+impl<RelayerId> Default for LegacyInboundLaneData<RelayerId> {
+	fn default() -> Self {
+		let full = bp_messages::InboundLaneData::default();
+		Self { relayers: full.relayers, last_confirmed_nonce: full.last_confirmed_nonce }
+	}
+}
+
+impl<RelayerId> LegacyInboundLaneData<RelayerId> {
+	pub fn last_delivered_nonce(self) -> MessageNonce {
+		bp_messages::InboundLaneData {
+			relayers: self.relayers,
+			last_confirmed_nonce: self.last_confirmed_nonce,
+			// we don't care about the state here
+			state: LaneState::Opened,
+		}
+		.last_delivered_nonce()
+	}
+}
+
+impl<RelayerId> From<LegacyInboundLaneData<RelayerId>> for UnrewardedRelayersState {
+	fn from(value: LegacyInboundLaneData<RelayerId>) -> Self {
+		(&bp_messages::InboundLaneData {
+			relayers: value.relayers,
+			last_confirmed_nonce: value.last_confirmed_nonce,
+			// we don't care about the state here
+			state: LaneState::Opened,
+		})
+			.into()
+	}
+}
+
+/// Bizinikiwi client as Bizinikiwi messages target.
+pub struct BizinikiwiMessagesTarget<P: BizinikiwiMessageLane, SourceClnt, TargetClnt> {
+	target_client: TargetClnt,
+	source_client: SourceClnt,
+	lane_id: P::LaneId,
+	relayer_id_at_source: AccountIdOf<P::SourceChain>,
+	transaction_params: Option<TransactionParams<AccountKeyPairOf<P::TargetChain>>>,
+	source_to_target_headers_relay: Option<Arc<dyn OnDemandRelay<P::SourceChain, P::TargetChain>>>,
+}
+
+impl<P, SourceClnt, TargetClnt> BizinikiwiMessagesTarget<P, SourceClnt, TargetClnt>
+where
+	P: BizinikiwiMessageLane,
+	TargetClnt: Client<P::TargetChain>,
+{
+	/// Create new Bizinikiwi headers target.
+	pub fn new(
+		target_client: TargetClnt,
+		source_client: SourceClnt,
+		lane_id: P::LaneId,
+		relayer_id_at_source: AccountIdOf<P::SourceChain>,
+		transaction_params: Option<TransactionParams<AccountKeyPairOf<P::TargetChain>>>,
+		source_to_target_headers_relay: Option<
+			Arc<dyn OnDemandRelay<P::SourceChain, P::TargetChain>>,
+		>,
+	) -> Self {
+		BizinikiwiMessagesTarget {
+			target_client,
+			source_client,
+			lane_id,
+			relayer_id_at_source,
+			transaction_params,
+			source_to_target_headers_relay,
+		}
+	}
+
+	/// Read inbound lane state from the on-chain storage at given block.
+	async fn inbound_lane_data(
+		&self,
+		id: TargetHeaderIdOf<MessageLaneAdapter<P>>,
+	) -> Result<Option<LegacyInboundLaneData<AccountIdOf<P::SourceChain>>>, BizinikiwiError> {
+		self.target_client
+			.storage_value(
+				id.hash(),
+				inbound_lane_data_key(
+					P::SourceChain::WITH_CHAIN_MESSAGES_PALLET_NAME,
+					&self.lane_id,
+				),
+			)
+			.await
+	}
+
+	/// Ensure that the messages pezpallet at target chain is active.
+	async fn ensure_pallet_active(&self) -> Result<(), BizinikiwiError> {
+		ensure_messages_pallet_active::<P::TargetChain, P::SourceChain, _>(&self.target_client)
+			.await
+	}
+}
+
+impl<P: BizinikiwiMessageLane, SourceClnt: Clone, TargetClnt: Clone> Clone
+	for BizinikiwiMessagesTarget<P, SourceClnt, TargetClnt>
+{
+	fn clone(&self) -> Self {
+		Self {
+			target_client: self.target_client.clone(),
+			source_client: self.source_client.clone(),
+			lane_id: self.lane_id,
+			relayer_id_at_source: self.relayer_id_at_source.clone(),
+			transaction_params: self.transaction_params.clone(),
+			source_to_target_headers_relay: self.source_to_target_headers_relay.clone(),
+		}
+	}
+}
+
+#[async_trait]
+impl<
+		P: BizinikiwiMessageLane,
+		SourceClnt: Client<P::SourceChain>,
+		TargetClnt: Client<P::TargetChain>,
+	> RelayClient for BizinikiwiMessagesTarget<P, SourceClnt, TargetClnt>
+{
+	type Error = BizinikiwiError;
+
+	async fn reconnect(&mut self) -> Result<(), BizinikiwiError> {
+		// since the client calls RPC methods on both sides, we need to reconnect both
+		self.target_client.reconnect().await?;
+		self.source_client.reconnect().await?;
+
+		// call reconnect on on-demand headers relay, because we may use different chains there
+		// and the error that has lead to reconnect may have came from those other chains
+		// (see `require_source_header_on_target`)
+		//
+		// this may lead to multiple reconnects to the same node during the same call and it
+		// needs to be addressed in the future
+		// TODO: https://github.com/pezkuwichain/kurdistan-sdk/issues/82
+		if let Some(ref mut source_to_target_headers_relay) = self.source_to_target_headers_relay {
+			source_to_target_headers_relay.reconnect().await?;
+		}
+
+		Ok(())
+	}
+}
+
+#[async_trait]
+impl<
+		P: BizinikiwiMessageLane,
+		SourceClnt: Client<P::SourceChain>,
+		TargetClnt: Client<P::TargetChain>,
+	> TargetClient<MessageLaneAdapter<P>> for BizinikiwiMessagesTarget<P, SourceClnt, TargetClnt>
+where
+	AccountIdOf<P::TargetChain>: From<<AccountKeyPairOf<P::TargetChain> as Pair>::Public>,
+	BalanceOf<P::SourceChain>: TryFrom<BalanceOf<P::TargetChain>>,
+{
+	type BatchTransaction =
+		BatchProofTransaction<P::TargetChain, P::SourceChain, P::TargetBatchCallBuilder>;
+	type TransactionTracker = TransactionTracker<P::TargetChain, TargetClnt>;
+
+	async fn state(&self) -> Result<TargetClientState<MessageLaneAdapter<P>>, BizinikiwiError> {
+		// we can't continue to deliver confirmations if source node is out of sync, because
+		// it may have already received confirmations that we're going to deliver
+		//
+		// we can't continue to deliver messages if target node is out of sync, because
+		// it may have already received (some of) messages that we're going to deliver
+		self.source_client.ensure_synced().await?;
+		self.target_client.ensure_synced().await?;
+		// we can't relay messages if messages pezpallet at target chain is halted
+		self.ensure_pallet_active().await?;
+
+		read_client_state_from_both_chains(&self.target_client, &self.source_client).await
+	}
+
+	async fn latest_received_nonce(
+		&self,
+		id: TargetHeaderIdOf<MessageLaneAdapter<P>>,
+	) -> Result<(TargetHeaderIdOf<MessageLaneAdapter<P>>, MessageNonce), BizinikiwiError> {
+		// lane data missing from the storage is fine until first message is received
+		let latest_received_nonce = self
+			.inbound_lane_data(id)
+			.await?
+			.map(|data| data.last_delivered_nonce())
+			.unwrap_or(0);
+		Ok((id, latest_received_nonce))
+	}
+
+	async fn latest_confirmed_received_nonce(
+		&self,
+		id: TargetHeaderIdOf<MessageLaneAdapter<P>>,
+	) -> Result<(TargetHeaderIdOf<MessageLaneAdapter<P>>, MessageNonce), BizinikiwiError> {
+		// lane data missing from the storage is fine until first message is received
+		let last_confirmed_nonce = self
+			.inbound_lane_data(id)
+			.await?
+			.map(|data| data.last_confirmed_nonce)
+			.unwrap_or(0);
+		Ok((id, last_confirmed_nonce))
+	}
+
+	async fn unrewarded_relayers_state(
+		&self,
+		id: TargetHeaderIdOf<MessageLaneAdapter<P>>,
+	) -> Result<(TargetHeaderIdOf<MessageLaneAdapter<P>>, UnrewardedRelayersState), BizinikiwiError>
+	{
+		let inbound_lane_data =
+			self.inbound_lane_data(id).await?.unwrap_or(LegacyInboundLaneData::default());
+		Ok((id, inbound_lane_data.into()))
+	}
+
+	async fn prove_messages_receiving(
+		&self,
+		id: TargetHeaderIdOf<MessageLaneAdapter<P>>,
+	) -> Result<
+		(
+			TargetHeaderIdOf<MessageLaneAdapter<P>>,
+			<MessageLaneAdapter<P> as MessageLane>::MessagesReceivingProof,
+		),
+		BizinikiwiError,
+	> {
+		let (id, relayers_state) = self.unrewarded_relayers_state(id).await?;
+		let storage_keys = vec![inbound_lane_data_key(
+			P::SourceChain::WITH_CHAIN_MESSAGES_PALLET_NAME,
+			&self.lane_id,
+		)];
+
+		let storage_proof =
+			self.target_client.prove_storage(id.hash(), storage_keys.clone()).await?;
+		let proof = FromBridgedChainMessagesDeliveryProof {
+			bridged_header_hash: id.1,
+			storage_proof: to_raw_storage_proof::<P::TargetChain>(storage_proof),
+			lane: self.lane_id,
+		};
+		Ok((id, (relayers_state, proof)))
+	}
+
+	async fn submit_messages_proof(
+		&self,
+		maybe_batch_tx: Option<Self::BatchTransaction>,
+		_generated_at_header: SourceHeaderIdOf<MessageLaneAdapter<P>>,
+		nonces: RangeInclusive<MessageNonce>,
+		proof: <MessageLaneAdapter<P> as MessageLane>::MessagesProof,
+	) -> Result<NoncesSubmitArtifacts<Self::TransactionTracker>, BizinikiwiError> {
+		let messages_proof_call = make_messages_delivery_call::<P>(
+			self.relayer_id_at_source.clone(),
+			proof.1.nonces_start..=proof.1.nonces_end,
+			proof,
+			maybe_batch_tx.is_none(),
+		);
+		let final_call = match maybe_batch_tx {
+			Some(batch_tx) => batch_tx.append_call_and_build(messages_proof_call),
+			None => messages_proof_call,
+		};
+
+		let transaction_params = self.transaction_params.clone().map(Ok).unwrap_or_else(|| {
+			// this error shall never happen in practice, so it not deserves
+			// a separate error variant
+			Err(BizinikiwiError::Custom(format!(
+				"Cannot sign transaction of {} chain",
+				P::TargetChain::NAME,
+			)))
+		})?;
+		let tx_tracker = self
+			.target_client
+			.submit_and_watch_signed_extrinsic(
+				&transaction_params.signer,
+				move |best_block_id, transaction_nonce| {
+					Ok(UnsignedTransaction::new(final_call.into(), transaction_nonce)
+						.era(TransactionEra::new(best_block_id, transaction_params.mortality)))
+				},
+			)
+			.await?;
+		Ok(NoncesSubmitArtifacts { nonces, tx_tracker })
+	}
+
+	async fn require_source_header_on_target(
+		&self,
+		id: SourceHeaderIdOf<MessageLaneAdapter<P>>,
+	) -> Result<Option<Self::BatchTransaction>, BizinikiwiError> {
+		if let Some(ref source_to_target_headers_relay) = self.source_to_target_headers_relay {
+			if let Some(batch_tx) =
+				BatchProofTransaction::new(source_to_target_headers_relay.clone(), id.0).await?
+			{
+				return Ok(Some(batch_tx));
+			}
+
+			source_to_target_headers_relay.require_more_headers(id.0).await;
+		}
+
+		Ok(None)
+	}
+}
+
+/// Make messages delivery call from given proof.
+fn make_messages_delivery_call<P: BizinikiwiMessageLane>(
+	relayer_id_at_source: AccountIdOf<P::SourceChain>,
+	nonces: RangeInclusive<MessageNonce>,
+	proof: BizinikiwiMessagesProof<P::SourceChain, P::LaneId>,
+	trace_call: bool,
+) -> CallOf<P::TargetChain> {
+	let messages_count = nonces.end() - nonces.start() + 1;
+	let dispatch_weight = proof.0;
+	P::ReceiveMessagesProofCallBuilder::build_receive_messages_proof_call(
+		relayer_id_at_source,
+		proof,
+		messages_count as _,
+		dispatch_weight,
+		trace_call,
+	)
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use bp_messages::{DeliveredMessages, UnrewardedRelayer};
+	use codec::Encode;
+
+	#[test]
+	fn inbound_lane_data_wrapper_is_compatible() {
+		let bytes_without_state =
+			vec![4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0];
+		let bytes_with_state = {
+			// add state byte `bp_messages::LaneState::Opened`
+			let mut b = bytes_without_state.clone();
+			b.push(0);
+			b
+		};
+
+		let full = bp_messages::InboundLaneData::<u8> {
+			relayers: vec![UnrewardedRelayer {
+				relayer: Default::default(),
+				messages: DeliveredMessages { begin: 2, end: 5 },
+			}]
+			.into_iter()
+			.collect(),
+			last_confirmed_nonce: 6,
+			state: bp_messages::LaneState::Opened,
+		};
+		assert_eq!(full.encode(), bytes_with_state);
+		assert_ne!(full.encode(), bytes_without_state);
+
+		// decode from `bytes_with_state`
+		let decoded: LegacyInboundLaneData<u8> =
+			Decode::decode(&mut &bytes_with_state[..]).unwrap();
+		assert_eq!(full.relayers, decoded.relayers);
+		assert_eq!(full.last_confirmed_nonce, decoded.last_confirmed_nonce);
+		assert_eq!(full.last_delivered_nonce(), decoded.last_delivered_nonce());
+
+		// decode from `bytes_without_state`
+		let decoded: LegacyInboundLaneData<u8> =
+			Decode::decode(&mut &bytes_without_state[..]).unwrap();
+		assert_eq!(full.relayers, decoded.relayers);
+		assert_eq!(full.last_confirmed_nonce, decoded.last_confirmed_nonce);
+		assert_eq!(full.last_delivered_nonce(), decoded.last_delivered_nonce());
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/on_demand/headers.rs

@@ -0,0 +1,569 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! On-demand Bizinikiwi -> Bizinikiwi header finality relay.
+
+use crate::finality::SubmitFinalityProofCallBuilder;
+
+use async_std::sync::{Arc, Mutex};
+use async_trait::async_trait;
+use bp_header_pez_chain::ConsensusLogReader;
+use pezbp_runtime::HeaderIdProvider;
+use futures::{select, FutureExt};
+use num_traits::{One, Saturating, Zero};
+use pezsp_runtime::traits::Header;
+
+use pez_finality_relay::{FinalitySyncParams, HeadersToRelay, TargetClient as FinalityTargetClient};
+use relay_bizinikiwi_client::{
+	AccountIdOf, AccountKeyPairOf, BlockNumberOf, CallOf, Chain, Client, Error as BizinikiwiError,
+	HeaderIdOf,
+};
+use relay_utils::{
+	metrics::MetricsParams, relay_loop::Client as RelayClient, FailedClient, MaybeConnectionError,
+	STALL_TIMEOUT,
+};
+
+use crate::{
+	finality::{
+		source::{RequiredHeaderNumberRef, BizinikiwiFinalitySource},
+		target::BizinikiwiFinalityTarget,
+		BizinikiwiFinalitySyncPipeline, RECENT_FINALITY_PROOFS_LIMIT,
+	},
+	finality_base::engine::Engine,
+	on_demand::OnDemandRelay,
+	TransactionParams,
+};
+
+/// On-demand Bizinikiwi <-> Bizinikiwi header finality relay.
+///
+/// This relay may be requested to sync more headers, whenever some other relay (e.g. messages
+/// relay) needs it to continue its regular work. When enough headers are relayed, on-demand stops
+/// syncing headers.
+#[derive(Clone)]
+pub struct OnDemandHeadersRelay<P: BizinikiwiFinalitySyncPipeline, SourceClnt, TargetClnt> {
+	/// Relay task name.
+	relay_task_name: String,
+	/// Shared reference to maximal required finalized header number.
+	required_header_number: RequiredHeaderNumberRef<P::SourceChain>,
+	/// Client of the source chain.
+	source_client: SourceClnt,
+	/// Client of the target chain.
+	target_client: TargetClnt,
+}
+
+impl<
+		P: BizinikiwiFinalitySyncPipeline,
+		SourceClnt: Client<P::SourceChain>,
+		TargetClnt: Client<P::TargetChain>,
+	> OnDemandHeadersRelay<P, SourceClnt, TargetClnt>
+{
+	/// Create new on-demand headers relay.
+	///
+	/// If `metrics_params` is `Some(_)`, the metrics of the finality relay are registered.
+	/// Otherwise, all required metrics must be exposed outside of this method.
+	pub fn new(
+		source_client: SourceClnt,
+		target_client: TargetClnt,
+		target_transaction_params: TransactionParams<AccountKeyPairOf<P::TargetChain>>,
+		headers_to_relay: HeadersToRelay,
+		metrics_params: Option<MetricsParams>,
+	) -> Self
+	where
+		AccountIdOf<P::TargetChain>:
+			From<<AccountKeyPairOf<P::TargetChain> as pezsp_core::Pair>::Public>,
+	{
+		let required_header_number = Arc::new(Mutex::new(Zero::zero()));
+		let this = OnDemandHeadersRelay {
+			relay_task_name: on_demand_headers_relay_name::<P::SourceChain, P::TargetChain>(),
+			required_header_number: required_header_number.clone(),
+			source_client: source_client.clone(),
+			target_client: target_client.clone(),
+		};
+		async_std::task::spawn(async move {
+			background_task::<P>(
+				source_client,
+				target_client,
+				target_transaction_params,
+				headers_to_relay,
+				required_header_number,
+				metrics_params,
+			)
+			.await;
+		});
+
+		this
+	}
+}
+
+#[async_trait]
+impl<
+		P: BizinikiwiFinalitySyncPipeline,
+		SourceClnt: Client<P::SourceChain>,
+		TargetClnt: Client<P::TargetChain>,
+	> OnDemandRelay<P::SourceChain, P::TargetChain>
+	for OnDemandHeadersRelay<P, SourceClnt, TargetClnt>
+{
+	async fn reconnect(&self) -> Result<(), BizinikiwiError> {
+		// using clone is fine here (to avoid mut requirement), because clone on Client clones
+		// internal references
+		self.source_client.clone().reconnect().await?;
+		self.target_client.clone().reconnect().await
+	}
+
+	async fn require_more_headers(&self, required_header: BlockNumberOf<P::SourceChain>) {
+		let mut required_header_number = self.required_header_number.lock().await;
+		if required_header > *required_header_number {
+			tracing::trace!(
+				target: "bridge",
+				relay_task_name=%self.relay_task_name,
+				source=%P::SourceChain::NAME,
+				%required_header,
+				"More headers required. Going to sync up"
+			);
+
+			*required_header_number = required_header;
+		}
+	}
+
+	async fn prove_header(
+		&self,
+		required_header: BlockNumberOf<P::SourceChain>,
+	) -> Result<(HeaderIdOf<P::SourceChain>, Vec<CallOf<P::TargetChain>>), BizinikiwiError> {
+		const MAX_ITERATIONS: u32 = 4;
+		let mut iterations = 0;
+		let mut current_required_header = required_header;
+		loop {
+			// first find proper header (either `current_required_header`) or its descendant
+			let finality_source =
+				BizinikiwiFinalitySource::<P, _>::new(self.source_client.clone(), None);
+			let (header, mut proof) =
+				finality_source.prove_block_finality(current_required_header).await?;
+			let header_id = header.id();
+
+			// verify and optimize justification before including it into the call
+			let context = P::FinalityEngine::verify_and_optimize_proof(
+				&self.target_client,
+				&header,
+				&mut proof,
+			)
+			.await?;
+
+			// now we have the header and its proof, but we want to minimize our losses, so let's
+			// check if we'll get the full refund for submitting this header
+			let check_result = P::FinalityEngine::check_max_expected_call_limits(&header, &proof);
+			if check_result.is_weight_limit_exceeded || check_result.extra_size != 0 {
+				iterations += 1;
+				current_required_header = header_id.number().saturating_add(One::one());
+				if iterations < MAX_ITERATIONS {
+					tracing::debug!(
+						target: "bridge",
+						relay_task_name=%self.relay_task_name,
+						source=%P::SourceChain::NAME,
+						%required_header,
+						?header_id,
+						?check_result,
+						"Requested to prove head. Selected to prove head. But it exceeds limits. \
+						Going to select next header"
+					);
+
+					continue;
+				}
+			}
+
+			tracing::debug!(
+				target: "bridge",
+				relay_task_name=%self.relay_task_name,
+				source=%P::SourceChain::NAME,
+				%required_header,
+				?header_id,
+				%iterations,
+				"Requested to prove head. Selected to prove head (after iterations)",
+			);
+
+			// and then craft the submit-proof call
+			let call = P::SubmitFinalityProofCallBuilder::build_submit_finality_proof_call(
+				header, proof, false, context,
+			);
+
+			return Ok((header_id, vec![call]));
+		}
+	}
+}
+
+/// Background task that is responsible for starting headers relay.
+async fn background_task<P: BizinikiwiFinalitySyncPipeline>(
+	source_client: impl Client<P::SourceChain>,
+	target_client: impl Client<P::TargetChain>,
+	target_transaction_params: TransactionParams<AccountKeyPairOf<P::TargetChain>>,
+	headers_to_relay: HeadersToRelay,
+	required_header_number: RequiredHeaderNumberRef<P::SourceChain>,
+	metrics_params: Option<MetricsParams>,
+) where
+	AccountIdOf<P::TargetChain>: From<<AccountKeyPairOf<P::TargetChain> as pezsp_core::Pair>::Public>,
+{
+	let relay_task_name = on_demand_headers_relay_name::<P::SourceChain, P::TargetChain>();
+	let target_transactions_mortality = target_transaction_params.mortality;
+	let mut finality_source = BizinikiwiFinalitySource::<P, _>::new(
+		source_client.clone(),
+		Some(required_header_number.clone()),
+	);
+	let mut finality_target =
+		BizinikiwiFinalityTarget::new(target_client.clone(), target_transaction_params);
+	let mut latest_non_mandatory_at_source = Zero::zero();
+
+	let mut restart_relay = true;
+	let pez_finality_relay_task = futures::future::Fuse::terminated();
+	futures::pin_mut!(pez_finality_relay_task);
+
+	loop {
+		select! {
+			_ = async_std::task::sleep(P::TargetChain::AVERAGE_BLOCK_INTERVAL).fuse() => {},
+			_ = pez_finality_relay_task => {
+				// this should never happen in practice given the current code
+				restart_relay = true;
+			},
+		}
+
+		// read best finalized source header number from source
+		let best_finalized_source_header_at_source =
+			best_finalized_source_header_at_source(&finality_source, &relay_task_name).await;
+		if matches!(best_finalized_source_header_at_source, Err(ref e) if e.is_connection_error()) {
+			relay_utils::relay_loop::reconnect_failed_client(
+				FailedClient::Source,
+				relay_utils::relay_loop::RECONNECT_DELAY,
+				&mut finality_source,
+				&mut finality_target,
+			)
+			.await;
+			continue;
+		}
+
+		// read best finalized source header number from target
+		let best_finalized_source_header_at_target =
+			best_finalized_source_header_at_target::<P, _>(&finality_target, &relay_task_name)
+				.await;
+		if matches!(best_finalized_source_header_at_target, Err(ref e) if e.is_connection_error()) {
+			relay_utils::relay_loop::reconnect_failed_client(
+				FailedClient::Target,
+				relay_utils::relay_loop::RECONNECT_DELAY,
+				&mut finality_source,
+				&mut finality_target,
+			)
+			.await;
+			continue;
+		}
+
+		// submit mandatory header if some headers are missing
+		let best_finalized_source_header_at_source_fmt =
+			format!("{best_finalized_source_header_at_source:?}");
+		let best_finalized_source_header_at_target_fmt =
+			format!("{best_finalized_source_header_at_target:?}");
+		let required_header_number_value = *required_header_number.lock().await;
+		let mandatory_scan_range = mandatory_headers_scan_range::<P::SourceChain>(
+			best_finalized_source_header_at_source.ok(),
+			best_finalized_source_header_at_target.ok(),
+			required_header_number_value,
+		)
+		.await;
+
+		tracing::trace!(
+			target: "bridge",
+			%relay_task_name,
+			?required_header_number_value,
+			?best_finalized_source_header_at_source_fmt,
+			?best_finalized_source_header_at_target_fmt,
+			?mandatory_scan_range,
+			"Mandatory headers scan range"
+		);
+
+		if let Some(mandatory_scan_range) = mandatory_scan_range {
+			let relay_mandatory_header_result = relay_mandatory_header_from_range(
+				&finality_source,
+				&required_header_number,
+				best_finalized_source_header_at_target_fmt,
+				(
+					std::cmp::max(mandatory_scan_range.0, latest_non_mandatory_at_source),
+					mandatory_scan_range.1,
+				),
+				&relay_task_name,
+			)
+			.await;
+			match relay_mandatory_header_result {
+				Ok(true) => (),
+				Ok(false) => {
+					// there are no (or we don't need to relay them) mandatory headers in the range
+					// => to avoid scanning the same headers over and over again, remember that
+					latest_non_mandatory_at_source = mandatory_scan_range.1;
+
+					tracing::trace!(
+						target: "bridge",
+						%relay_task_name,
+						source=%P::SourceChain::NAME,
+						?mandatory_scan_range,
+						"No mandatory headers in the range"
+					);
+				},
+				Err(e) => {
+					tracing::warn!(
+						target: "bridge",
+						error=?e,
+						%relay_task_name,
+						source=%P::SourceChain::NAME,
+						?mandatory_scan_range,
+						"Failed to scan mandatory headers range"
+					);
+
+					if e.is_connection_error() {
+						relay_utils::relay_loop::reconnect_failed_client(
+							FailedClient::Source,
+							relay_utils::relay_loop::RECONNECT_DELAY,
+							&mut finality_source,
+							&mut finality_target,
+						)
+						.await;
+						continue;
+					}
+				},
+			}
+		}
+
+		// start/restart relay
+		if restart_relay {
+			let stall_timeout = relay_bizinikiwi_client::transaction_stall_timeout(
+				target_transactions_mortality,
+				P::TargetChain::AVERAGE_BLOCK_INTERVAL,
+				STALL_TIMEOUT,
+			);
+
+			tracing::info!(
+				target: "bridge",
+				%relay_task_name,
+				?headers_to_relay,
+				?target_transactions_mortality,
+				stall_timeout_as_mins=%stall_timeout.as_secs_f64() / 60.0f64,
+				?stall_timeout,
+				"Starting on-demand headers relay task"
+			);
+
+			pez_finality_relay_task.set(
+				pez_finality_relay::run(
+					finality_source.clone(),
+					finality_target.clone(),
+					FinalitySyncParams {
+						tick: std::cmp::max(
+							P::SourceChain::AVERAGE_BLOCK_INTERVAL,
+							P::TargetChain::AVERAGE_BLOCK_INTERVAL,
+						),
+						recent_finality_proofs_limit: RECENT_FINALITY_PROOFS_LIMIT,
+						stall_timeout,
+						headers_to_relay,
+					},
+					metrics_params.clone().unwrap_or_else(MetricsParams::disabled),
+					futures::future::pending(),
+				)
+				.fuse(),
+			);
+
+			restart_relay = false;
+		}
+	}
+}
+
+/// Returns `Some()` with inclusive range of headers which must be scanned for mandatory headers
+/// and the first of such headers must be submitted to the target node.
+async fn mandatory_headers_scan_range<C: Chain>(
+	best_finalized_source_header_at_source: Option<C::BlockNumber>,
+	best_finalized_source_header_at_target: Option<C::BlockNumber>,
+	required_header_number: BlockNumberOf<C>,
+) -> Option<(C::BlockNumber, C::BlockNumber)> {
+	// if we have been unable to read header number from the target, then let's assume
+	// that it is the same as required header number. Otherwise we risk submitting
+	// unneeded transactions
+	let best_finalized_source_header_at_target =
+		best_finalized_source_header_at_target.unwrap_or(required_header_number);
+
+	// if we have been unable to read header number from the source, then let's assume
+	// that it is the same as at the target
+	let best_finalized_source_header_at_source =
+		best_finalized_source_header_at_source.unwrap_or(best_finalized_source_header_at_target);
+
+	// if relay is already asked to sync more headers than we have at source, don't do anything yet
+	if required_header_number >= best_finalized_source_header_at_source {
+		return None;
+	}
+
+	Some((
+		best_finalized_source_header_at_target + One::one(),
+		best_finalized_source_header_at_source,
+	))
+}
+
+/// Try to find mandatory header in the inclusive headers range and, if one is found, ask to relay
+/// it.
+///
+/// Returns `true` if header was found and (asked to be) relayed and `false` otherwise.
+async fn relay_mandatory_header_from_range<P, SourceClnt>(
+	finality_source: &BizinikiwiFinalitySource<P, SourceClnt>,
+	required_header_number: &RequiredHeaderNumberRef<P::SourceChain>,
+	best_finalized_source_header_at_target: String,
+	range: (BlockNumberOf<P::SourceChain>, BlockNumberOf<P::SourceChain>),
+	relay_task_name: &str,
+) -> Result<bool, relay_bizinikiwi_client::Error>
+where
+	P: BizinikiwiFinalitySyncPipeline,
+	SourceClnt: Client<P::SourceChain>,
+{
+	// search for mandatory header first
+	let mandatory_source_header_number =
+		find_mandatory_header_in_range(finality_source, range).await?;
+
+	// if there are no mandatory headers - we have nothing to do
+	let mandatory_source_header_number = match mandatory_source_header_number {
+		Some(mandatory_source_header_number) => mandatory_source_header_number,
+		None => return Ok(false),
+	};
+
+	// `find_mandatory_header` call may take a while => check if `required_header_number` is still
+	// less than our `mandatory_source_header_number` before logging anything
+	let mut required_header_number = required_header_number.lock().await;
+	if *required_header_number >= mandatory_source_header_number {
+		return Ok(false);
+	}
+
+	tracing::trace!(
+		target: "bridge",
+		%relay_task_name,
+		source=%P::SourceChain::NAME,
+		%best_finalized_source_header_at_target,
+		at_source=%range.1,
+		%mandatory_source_header_number,
+		"Too many headers missing at target. Going to sync up to the mandatory"
+	);
+
+	*required_header_number = mandatory_source_header_number;
+	Ok(true)
+}
+
+/// Read best finalized source block number from source client.
+///
+/// Returns `None` if we have failed to read the number.
+async fn best_finalized_source_header_at_source<P, SourceClnt>(
+	finality_source: &BizinikiwiFinalitySource<P, SourceClnt>,
+	relay_task_name: &str,
+) -> Result<BlockNumberOf<P::SourceChain>, relay_bizinikiwi_client::Error>
+where
+	P: BizinikiwiFinalitySyncPipeline,
+	SourceClnt: Client<P::SourceChain>,
+{
+	finality_source.on_chain_best_finalized_block_number().await.map_err(|error| {
+		tracing::error!(
+			target: "bridge",
+			?error,
+			%relay_task_name,
+			"Failed to read best finalized source header from source"
+		);
+
+		error
+	})
+}
+
+/// Read best finalized source block number from target client.
+///
+/// Returns `None` if we have failed to read the number.
+async fn best_finalized_source_header_at_target<P, TargetClnt>(
+	finality_target: &BizinikiwiFinalityTarget<P, TargetClnt>,
+	relay_task_name: &str,
+) -> Result<
+	BlockNumberOf<P::SourceChain>,
+	<BizinikiwiFinalityTarget<P, TargetClnt> as RelayClient>::Error,
+>
+where
+	P: BizinikiwiFinalitySyncPipeline,
+	TargetClnt: Client<P::TargetChain>,
+	AccountIdOf<P::TargetChain>: From<<AccountKeyPairOf<P::TargetChain> as pezsp_core::Pair>::Public>,
+{
+	finality_target
+		.best_finalized_source_block_id()
+		.await
+		.map_err(|error| {
+			tracing::error!(
+				target: "bridge",
+				?error,
+				%relay_task_name,
+				"Failed to read best finalized source header from target"
+			);
+
+			error
+		})
+		.map(|id| id.0)
+}
+
+/// Read first mandatory header in given inclusive range.
+///
+/// Returns `Ok(None)` if there were no mandatory headers in the range.
+async fn find_mandatory_header_in_range<P, SourceClnt>(
+	finality_source: &BizinikiwiFinalitySource<P, SourceClnt>,
+	range: (BlockNumberOf<P::SourceChain>, BlockNumberOf<P::SourceChain>),
+) -> Result<Option<BlockNumberOf<P::SourceChain>>, relay_bizinikiwi_client::Error>
+where
+	P: BizinikiwiFinalitySyncPipeline,
+	SourceClnt: Client<P::SourceChain>,
+{
+	let mut current = range.0;
+	while current <= range.1 {
+		let header = finality_source.client().header_by_number(current).await?;
+		if <P::FinalityEngine as Engine<P::SourceChain>>::ConsensusLogReader::schedules_authorities_change(
+			header.digest(),
+		) {
+			return Ok(Some(current))
+		}
+
+		current += One::one();
+	}
+
+	Ok(None)
+}
+
+/// On-demand headers relay task name.
+fn on_demand_headers_relay_name<SourceChain: Chain, TargetChain: Chain>() -> String {
+	format!("{}-to-{}-on-demand-headers", SourceChain::NAME, TargetChain::NAME)
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use relay_bizinikiwi_client::test_chain::TestChain;
+
+	const AT_SOURCE: Option<BlockNumberOf<TestChain>> = Some(10);
+	const AT_TARGET: Option<BlockNumberOf<TestChain>> = Some(1);
+
+	#[async_std::test]
+	async fn mandatory_headers_scan_range_selects_range_if_some_headers_are_missing() {
+		assert_eq!(
+			mandatory_headers_scan_range::<TestChain>(AT_SOURCE, AT_TARGET, 0,).await,
+			Some((AT_TARGET.unwrap() + 1, AT_SOURCE.unwrap())),
+		);
+	}
+
+	#[async_std::test]
+	async fn mandatory_headers_scan_range_selects_nothing_if_already_queued() {
+		assert_eq!(
+			mandatory_headers_scan_range::<TestChain>(AT_SOURCE, AT_TARGET, AT_SOURCE.unwrap(),)
+				.await,
+			None,
+		);
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/on_demand/mod.rs

@@ -0,0 +1,48 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Types and functions intended to ease adding of new Bizinikiwi -> Bizinikiwi
+//! on-demand pipelines.
+
+use async_trait::async_trait;
+use relay_bizinikiwi_client::{BlockNumberOf, CallOf, Chain, Error as BizinikiwiError, HeaderIdOf};
+
+pub mod headers;
+pub mod teyrchains;
+
+/// On-demand headers relay that is relaying finalizing headers only when requested.
+#[async_trait]
+pub trait OnDemandRelay<SourceChain: Chain, TargetChain: Chain>: Send + Sync {
+	/// Reconnect to source and target nodes.
+	async fn reconnect(&self) -> Result<(), BizinikiwiError>;
+
+	/// Ask relay to relay source header with given number  to the target chain.
+	///
+	/// Depending on implementation, on-demand relay may also relay `required_header` ancestors
+	/// (e.g. if they're mandatory), or its descendants. The request is considered complete if
+	/// the best avbailable header at the target chain has number that is larger than or equal
+	/// to the `required_header`.
+	async fn require_more_headers(&self, required_header: BlockNumberOf<SourceChain>);
+
+	/// Ask relay to prove source `required_header` to the `TargetChain`.
+	///
+	/// Returns number of header that is proved (it may be the `required_header` or one of its
+	/// descendants) and calls for delivering the proof.
+	async fn prove_header(
+		&self,
+		required_header: BlockNumberOf<SourceChain>,
+	) -> Result<(HeaderIdOf<SourceChain>, Vec<CallOf<TargetChain>>), BizinikiwiError>;
+}

pezbridges/relays/lib-bizinikiwi-relay/src/teyrchains/mod.rs

@@ -0,0 +1,108 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Types and functions intended to ease adding of new Bizinikiwi -> Bizinikiwi
+//! teyrchain finality proofs synchronization pipelines.
+
+use async_trait::async_trait;
+use bp_pezkuwi_core::teyrchains::{ParaHash, ParaHeadsProof, ParaId};
+use bp_teyrchains::{RelayBlockHash, RelayBlockHasher, RelayBlockNumber};
+use pezpallet_bridge_teyrchains::{Call as BridgeTeyrchainsCall, Config as BridgeTeyrchainsConfig};
+use relay_bizinikiwi_client::{
+	CallOf, Chain, ChainWithTransactions, HeaderIdOf, RelayChain, Teyrchain,
+};
+use std::{fmt::Debug, marker::PhantomData};
+use teyrchains_relay::TeyrchainsPipeline;
+
+pub mod source;
+pub mod target;
+
+/// Bizinikiwi -> Bizinikiwi teyrchain finality proofs synchronization pipeline.
+///
+/// This is currently restricted to the single teyrchain, because it is how it
+/// will be used (at least) initially.
+#[async_trait]
+pub trait BizinikiwiTeyrchainsPipeline: 'static + Clone + Debug + Send + Sync {
+	/// Headers of this teyrchain are submitted to the `Self::TargetChain`.
+	type SourceTeyrchain: Teyrchain;
+	/// Relay chain that is storing headers of `Self::SourceTeyrchain`.
+	type SourceRelayChain: RelayChain;
+	/// Target chain where `Self::SourceTeyrchain` headers are submitted.
+	type TargetChain: ChainWithTransactions;
+
+	/// How submit teyrchains heads call is built?
+	type SubmitTeyrchainHeadsCallBuilder: SubmitTeyrchainHeadsCallBuilder<Self>;
+}
+
+/// Adapter that allows all `BizinikiwiTeyrchainsPipeline` to act as `TeyrchainsPipeline`.
+#[derive(Clone, Debug)]
+pub struct TeyrchainsPipelineAdapter<P: BizinikiwiTeyrchainsPipeline> {
+	_phantom: PhantomData<P>,
+}
+
+impl<P: BizinikiwiTeyrchainsPipeline> TeyrchainsPipeline for TeyrchainsPipelineAdapter<P> {
+	type SourceTeyrchain = P::SourceTeyrchain;
+	type SourceRelayChain = P::SourceRelayChain;
+	type TargetChain = P::TargetChain;
+}
+
+/// Different ways of building `submit_teyrchain_heads` calls.
+pub trait SubmitTeyrchainHeadsCallBuilder<P: BizinikiwiTeyrchainsPipeline>:
+	'static + Send + Sync
+{
+	/// Given teyrchains and their heads proof, build call of `submit_teyrchain_heads`
+	/// function of bridge teyrchains module at the target chain.
+	fn build_submit_teyrchain_heads_call(
+		at_relay_block: HeaderIdOf<P::SourceRelayChain>,
+		teyrchains: Vec<(ParaId, ParaHash)>,
+		teyrchain_heads_proof: ParaHeadsProof,
+		is_free_execution_expected: bool,
+	) -> CallOf<P::TargetChain>;
+}
+
+/// Building `submit_teyrchain_heads` call when you have direct access to the target
+/// chain runtime.
+pub struct DirectSubmitTeyrchainHeadsCallBuilder<P, R, I> {
+	_phantom: PhantomData<(P, R, I)>,
+}
+
+impl<P, R, I> SubmitTeyrchainHeadsCallBuilder<P> for DirectSubmitTeyrchainHeadsCallBuilder<P, R, I>
+where
+	P: BizinikiwiTeyrchainsPipeline,
+	P::SourceRelayChain: Chain<Hash = RelayBlockHash, BlockNumber = RelayBlockNumber>,
+	R: BridgeTeyrchainsConfig<I> + Send + Sync,
+	I: 'static + Send + Sync,
+	R::BridgedChain: pezbp_runtime::Chain<
+		BlockNumber = RelayBlockNumber,
+		Hash = RelayBlockHash,
+		Hasher = RelayBlockHasher,
+	>,
+	CallOf<P::TargetChain>: From<BridgeTeyrchainsCall<R, I>>,
+{
+	fn build_submit_teyrchain_heads_call(
+		at_relay_block: HeaderIdOf<P::SourceRelayChain>,
+		teyrchains: Vec<(ParaId, ParaHash)>,
+		teyrchain_heads_proof: ParaHeadsProof,
+		_is_free_execution_expected: bool,
+	) -> CallOf<P::TargetChain> {
+		BridgeTeyrchainsCall::<R, I>::submit_teyrchain_heads {
+			at_relay_block: (at_relay_block.0, at_relay_block.1),
+			teyrchains,
+			teyrchain_heads_proof,
+		}
+		.into()
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/teyrchains/source.rs

@@ -0,0 +1,187 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Teyrchain heads source.
+
+use crate::{
+	proofs::to_raw_storage_proof,
+	teyrchains::{BizinikiwiTeyrchainsPipeline, TeyrchainsPipelineAdapter},
+};
+use async_std::sync::{Arc, Mutex};
+use async_trait::async_trait;
+use bp_pezkuwi_core::teyrchains::{ParaHash, ParaHead, ParaHeadsProof, ParaId};
+use pezbp_runtime::HeaderIdProvider;
+use bp_teyrchains::teyrchain_head_storage_key_at_source;
+use codec::Decode;
+use relay_bizinikiwi_client::{
+	is_ancient_block, Chain, Client, Error as BizinikiwiError, HeaderIdOf, HeaderOf, RelayChain,
+	TeyrchainBase,
+};
+use relay_utils::relay_loop::Client as RelayClient;
+use teyrchains_relay::teyrchains_loop::{AvailableHeader, SourceClient};
+
+/// Shared updatable reference to the maximal teyrchain header id that we want to sync from the
+/// source.
+pub type RequiredHeaderIdRef<C> = Arc<Mutex<AvailableHeader<HeaderIdOf<C>>>>;
+
+/// Bizinikiwi client as teyrchain heads source.
+#[derive(Clone)]
+pub struct TeyrchainsSource<P: BizinikiwiTeyrchainsPipeline, SourceRelayClnt> {
+	client: SourceRelayClnt,
+	max_head_id: RequiredHeaderIdRef<P::SourceTeyrchain>,
+}
+
+impl<P: BizinikiwiTeyrchainsPipeline, SourceRelayClnt: Client<P::SourceRelayChain>>
+	TeyrchainsSource<P, SourceRelayClnt>
+{
+	/// Creates new teyrchains source client.
+	pub fn new(
+		client: SourceRelayClnt,
+		max_head_id: RequiredHeaderIdRef<P::SourceTeyrchain>,
+	) -> Self {
+		TeyrchainsSource { client, max_head_id }
+	}
+
+	/// Returns reference to the underlying RPC client.
+	pub fn client(&self) -> &SourceRelayClnt {
+		&self.client
+	}
+
+	/// Return decoded head of given teyrchain.
+	pub async fn on_chain_para_head_id(
+		&self,
+		at_block: HeaderIdOf<P::SourceRelayChain>,
+	) -> Result<Option<HeaderIdOf<P::SourceTeyrchain>>, BizinikiwiError> {
+		let para_id = ParaId(P::SourceTeyrchain::TEYRCHAIN_ID);
+		let storage_key =
+			teyrchain_head_storage_key_at_source(P::SourceRelayChain::PARAS_PALLET_NAME, para_id);
+		let para_head: Option<ParaHead> =
+			self.client.storage_value(at_block.hash(), storage_key).await?;
+		let para_head = match para_head {
+			Some(para_head) => para_head,
+			None => return Ok(None),
+		};
+		let para_head: HeaderOf<P::SourceTeyrchain> = Decode::decode(&mut &para_head.0[..])?;
+		Ok(Some(para_head.id()))
+	}
+}
+
+#[async_trait]
+impl<P: BizinikiwiTeyrchainsPipeline, SourceRelayClnt: Client<P::SourceRelayChain>> RelayClient
+	for TeyrchainsSource<P, SourceRelayClnt>
+{
+	type Error = BizinikiwiError;
+
+	async fn reconnect(&mut self) -> Result<(), BizinikiwiError> {
+		self.client.reconnect().await
+	}
+}
+
+#[async_trait]
+impl<P: BizinikiwiTeyrchainsPipeline, SourceRelayClnt: Client<P::SourceRelayChain>>
+	SourceClient<TeyrchainsPipelineAdapter<P>> for TeyrchainsSource<P, SourceRelayClnt>
+where
+	P::SourceTeyrchain: Chain<Hash = ParaHash>,
+{
+	async fn ensure_synced(&self) -> Result<bool, Self::Error> {
+		match self.client.ensure_synced().await {
+			Ok(_) => Ok(true),
+			Err(BizinikiwiError::ClientNotSynced(_)) => Ok(false),
+			Err(e) => Err(e),
+		}
+	}
+
+	async fn teyrchain_head(
+		&self,
+		at_block: HeaderIdOf<P::SourceRelayChain>,
+	) -> Result<AvailableHeader<HeaderIdOf<P::SourceTeyrchain>>, Self::Error> {
+		// if requested relay header is ancient, then we don't even want to try to read the
+		// teyrchain head - we simply return `Unavailable`
+		let best_block_number = self.client.best_finalized_header_number().await?;
+		if is_ancient_block(at_block.number(), best_block_number) {
+			tracing::trace!(
+				target: "bridge",
+				source_relay_chain=%P::SourceRelayChain::NAME,
+				?at_block,
+				source=%P::SourceTeyrchain::NAME,
+				"Block is ancient. Cannot prove the header there"
+			);
+			return Ok(AvailableHeader::Unavailable);
+		}
+
+		// else - try to read head from the source client
+		let mut para_head_id = AvailableHeader::Missing;
+		if let Some(on_chain_para_head_id) = self.on_chain_para_head_id(at_block).await? {
+			// Never return head that is larger than requested. This way we'll never sync
+			// headers past `max_header_id`.
+			para_head_id = match *self.max_head_id.lock().await {
+				AvailableHeader::Unavailable => AvailableHeader::Unavailable,
+				AvailableHeader::Missing => {
+					// `max_header_id` is not set. There is no limit.
+					AvailableHeader::Available(on_chain_para_head_id)
+				},
+				AvailableHeader::Available(max_head_id) if on_chain_para_head_id >= max_head_id => {
+					// We report at most `max_header_id`.
+					AvailableHeader::Available(std::cmp::min(on_chain_para_head_id, max_head_id))
+				},
+				AvailableHeader::Available(_) => {
+					// the `max_head_id` is not yet available at the source chain => wait and avoid
+					// syncing extra headers
+					AvailableHeader::Unavailable
+				},
+			}
+		}
+
+		Ok(para_head_id)
+	}
+
+	async fn prove_teyrchain_head(
+		&self,
+		at_block: HeaderIdOf<P::SourceRelayChain>,
+	) -> Result<(ParaHeadsProof, ParaHash), Self::Error> {
+		let teyrchain = ParaId(P::SourceTeyrchain::TEYRCHAIN_ID);
+		let storage_key =
+			teyrchain_head_storage_key_at_source(P::SourceRelayChain::PARAS_PALLET_NAME, teyrchain);
+
+		let storage_proof =
+			self.client.prove_storage(at_block.hash(), vec![storage_key.clone()]).await?;
+
+		// why we're reading teyrchain head here once again (it has already been read at the
+		// `teyrchain_head`)? that's because `teyrchain_head` sometimes returns obsolete teyrchain
+		// head and loop sometimes asks to prove this obsolete head and gets other (actual) head
+		// instead
+		//
+		// => since we want to provide proper hashes in our `submit_teyrchain_heads` call, we're
+		// rereading actual value here
+		let teyrchain_head = self
+			.client
+			.storage_value::<ParaHead>(at_block.hash(), storage_key)
+			.await?
+			.ok_or_else(|| {
+				BizinikiwiError::Custom(format!(
+					"Failed to read expected teyrchain {teyrchain:?} head at {at_block:?}"
+				))
+			})?;
+		let teyrchain_head_hash = teyrchain_head.hash();
+
+		Ok((
+			ParaHeadsProof {
+				storage_proof: to_raw_storage_proof::<P::SourceRelayChain>(storage_proof),
+			},
+			teyrchain_head_hash,
+		))
+	}
+}

pezbridges/relays/lib-bizinikiwi-relay/src/teyrchains/target.rs

@@ -0,0 +1,233 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Teyrchain heads target.
+
+use crate::{
+	teyrchains::{
+		SubmitTeyrchainHeadsCallBuilder, BizinikiwiTeyrchainsPipeline, TeyrchainsPipelineAdapter,
+	},
+	TransactionParams,
+};
+
+use async_trait::async_trait;
+use bp_pezkuwi_core::{
+	teyrchains::{ParaHash, ParaHeadsProof, ParaId},
+	BlockNumber as RelayBlockNumber,
+};
+use pezbp_runtime::{
+	Chain as ChainBase, HeaderId, HeaderIdProvider, StorageDoubleMapKeyProvider,
+	StorageMapKeyProvider,
+};
+use bp_teyrchains::{
+	ImportedParaHeadsKeyProvider, ParaInfo, ParaStoredHeaderData, ParasInfoKeyProvider,
+};
+use relay_bizinikiwi_client::{
+	AccountIdOf, AccountKeyPairOf, BlockNumberOf, Chain, Client, Error as BizinikiwiError,
+	HeaderIdOf, RelayChain, TeyrchainBase, TransactionEra, TransactionTracker, UnsignedTransaction,
+};
+use relay_utils::relay_loop::Client as RelayClient;
+use pezsp_core::Pair;
+use pezsp_runtime::traits::Header;
+use teyrchains_relay::teyrchains_loop::TargetClient;
+
+/// Bizinikiwi client as teyrchain heads source.
+pub struct TeyrchainsTarget<P: BizinikiwiTeyrchainsPipeline, SourceClnt, TargetClnt> {
+	source_client: SourceClnt,
+	target_client: TargetClnt,
+	transaction_params: TransactionParams<AccountKeyPairOf<P::TargetChain>>,
+}
+
+impl<
+		P: BizinikiwiTeyrchainsPipeline,
+		SourceClnt: Client<P::SourceRelayChain>,
+		TargetClnt: Client<P::TargetChain>,
+	> TeyrchainsTarget<P, SourceClnt, TargetClnt>
+{
+	/// Creates new teyrchains target client.
+	pub fn new(
+		source_client: SourceClnt,
+		target_client: TargetClnt,
+		transaction_params: TransactionParams<AccountKeyPairOf<P::TargetChain>>,
+	) -> Self {
+		TeyrchainsTarget { source_client, target_client, transaction_params }
+	}
+
+	/// Returns reference to the underlying RPC client.
+	pub fn target_client(&self) -> &TargetClnt {
+		&self.target_client
+	}
+}
+
+impl<
+		P: BizinikiwiTeyrchainsPipeline,
+		SourceClnt: Client<P::SourceRelayChain>,
+		TargetClnt: Clone,
+	> Clone for TeyrchainsTarget<P, SourceClnt, TargetClnt>
+{
+	fn clone(&self) -> Self {
+		TeyrchainsTarget {
+			source_client: self.source_client.clone(),
+			target_client: self.target_client.clone(),
+			transaction_params: self.transaction_params.clone(),
+		}
+	}
+}
+
+#[async_trait]
+impl<
+		P: BizinikiwiTeyrchainsPipeline,
+		SourceClnt: Client<P::SourceRelayChain>,
+		TargetClnt: Client<P::TargetChain>,
+	> RelayClient for TeyrchainsTarget<P, SourceClnt, TargetClnt>
+{
+	type Error = BizinikiwiError;
+
+	async fn reconnect(&mut self) -> Result<(), BizinikiwiError> {
+		self.target_client.reconnect().await?;
+		self.source_client.reconnect().await?;
+		Ok(())
+	}
+}
+
+#[async_trait]
+impl<P, SourceClnt, TargetClnt> TargetClient<TeyrchainsPipelineAdapter<P>>
+	for TeyrchainsTarget<P, SourceClnt, TargetClnt>
+where
+	P: BizinikiwiTeyrchainsPipeline,
+	SourceClnt: Client<P::SourceRelayChain>,
+	TargetClnt: Client<P::TargetChain>,
+	AccountIdOf<P::TargetChain>: From<<AccountKeyPairOf<P::TargetChain> as Pair>::Public>,
+	P::SourceTeyrchain: ChainBase<Hash = ParaHash>,
+	P::SourceRelayChain: ChainBase<BlockNumber = RelayBlockNumber>,
+{
+	type TransactionTracker = TransactionTracker<P::TargetChain, TargetClnt>;
+
+	async fn best_block(&self) -> Result<HeaderIdOf<P::TargetChain>, Self::Error> {
+		let best_header = self.target_client.best_header().await?;
+		let best_id = best_header.id();
+
+		Ok(best_id)
+	}
+
+	async fn best_finalized_source_relay_chain_block(
+		&self,
+		at_block: &HeaderIdOf<P::TargetChain>,
+	) -> Result<HeaderIdOf<P::SourceRelayChain>, Self::Error> {
+		self.target_client
+			.state_call::<_, Option<HeaderIdOf<P::SourceRelayChain>>>(
+				at_block.hash(),
+				P::SourceRelayChain::BEST_FINALIZED_HEADER_ID_METHOD.into(),
+				(),
+			)
+			.await?
+			.map(Ok)
+			.unwrap_or(Err(BizinikiwiError::BridgePalletIsNotInitialized))
+	}
+
+	async fn free_source_relay_headers_interval(
+		&self,
+	) -> Result<Option<BlockNumberOf<P::SourceRelayChain>>, Self::Error> {
+		Ok(self
+			.target_client
+			.state_call(
+				self.target_client.best_header().await?.hash(),
+				P::SourceRelayChain::FREE_HEADERS_INTERVAL_METHOD.into(),
+				(),
+			)
+			.await
+			.unwrap_or_else(|e| {
+				tracing::info!(
+					target: "bridge",
+					error=?e,
+					methpd=%P::SourceRelayChain::FREE_HEADERS_INTERVAL_METHOD,
+					target=%P::TargetChain::NAME,
+					"Call has failed. Treating as `None`"
+				);
+				None
+			}))
+	}
+
+	async fn teyrchain_head(
+		&self,
+		at_block: HeaderIdOf<P::TargetChain>,
+	) -> Result<
+		Option<(HeaderIdOf<P::SourceRelayChain>, HeaderIdOf<P::SourceTeyrchain>)>,
+		Self::Error,
+	> {
+		// read best teyrchain head from the target bridge-teyrchains pezpallet
+		let storage_key = ParasInfoKeyProvider::final_key(
+			P::SourceRelayChain::WITH_CHAIN_BRIDGE_TEYRCHAINS_PALLET_NAME,
+			&P::SourceTeyrchain::TEYRCHAIN_ID.into(),
+		);
+		let storage_value: Option<ParaInfo> =
+			self.target_client.storage_value(at_block.hash(), storage_key).await?;
+		let para_info = match storage_value {
+			Some(para_info) => para_info,
+			None => return Ok(None),
+		};
+
+		// now we need to get full header ids. For source relay chain it is simple, because we
+		// are connected
+		let relay_header_id = self
+			.source_client
+			.header_by_number(para_info.best_head_hash.at_relay_block_number)
+			.await?
+			.id();
+
+		// for teyrchain, we need to read from the target chain runtime storage
+		let storage_key = ImportedParaHeadsKeyProvider::final_key(
+			P::SourceRelayChain::WITH_CHAIN_BRIDGE_TEYRCHAINS_PALLET_NAME,
+			&P::SourceTeyrchain::TEYRCHAIN_ID.into(),
+			&para_info.best_head_hash.head_hash,
+		);
+		let storage_value: Option<ParaStoredHeaderData> =
+			self.target_client.storage_value(at_block.hash(), storage_key).await?;
+		let para_head_number = match storage_value {
+			Some(para_head_data) =>
+				para_head_data.decode_teyrchain_head_data::<P::SourceTeyrchain>()?.number,
+			None => return Ok(None),
+		};
+
+		let para_head_id = HeaderId(para_head_number, para_info.best_head_hash.head_hash);
+		Ok(Some((relay_header_id, para_head_id)))
+	}
+
+	async fn submit_teyrchain_head_proof(
+		&self,
+		at_relay_block: HeaderIdOf<P::SourceRelayChain>,
+		updated_head_hash: ParaHash,
+		proof: ParaHeadsProof,
+		is_free_execution_expected: bool,
+	) -> Result<Self::TransactionTracker, Self::Error> {
+		let transaction_params = self.transaction_params.clone();
+		let call = P::SubmitTeyrchainHeadsCallBuilder::build_submit_teyrchain_heads_call(
+			at_relay_block,
+			vec![(ParaId(P::SourceTeyrchain::TEYRCHAIN_ID), updated_head_hash)],
+			proof,
+			is_free_execution_expected,
+		);
+		self.target_client
+			.submit_and_watch_signed_extrinsic(
+				&transaction_params.signer,
+				move |best_block_id, transaction_nonce| {
+					Ok(UnsignedTransaction::new(call.into(), transaction_nonce)
+						.era(TransactionEra::new(best_block_id, transaction_params.mortality)))
+				},
+			)
+			.await
+	}
+}

pezbridges/relays/messages/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+name = "pez-messages-relay"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+repository.workspace = true
+publish = false
+description = "Pezkuwi SDK component: messages relay"
+documentation = "https://docs.rs/pez-messages-relay"
+homepage = { workspace = true }
+
+[lints]
+workspace = true
+
+[dependencies]
+async-std = { features = ["attributes"], workspace = true }
+async-trait = { workspace = true }
+futures = { workspace = true }
+hex = { workspace = true, default-features = true }
+num-traits = { workspace = true, default-features = true }
+parking_lot = { workspace = true, default-features = true }
+tracing = { workspace = true }
+
+# Bridge Dependencies
+bp-messages = { workspace = true, default-features = true }
+pez-finality-relay = { workspace = true }
+relay-utils = { workspace = true }
+
+pezsp-arithmetic = { workspace = true, default-features = true }
+
+[dev-dependencies]
+pezsp-core = { workspace = true }
+
+[features]
+runtime-benchmarks = [
+  "bp-messages/runtime-benchmarks",
+  "pez-finality-relay/runtime-benchmarks",
+  "relay-utils/runtime-benchmarks",
+]

pezbridges/relays/messages/src/lib.rs

@@ -0,0 +1,41 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Relaying [`pezpallet-bridge-messages`](../pezpallet_bridge_messages/index.html) application specific
+//! data. Message lane allows sending arbitrary messages between bridged chains. This
+//! module provides entrypoint that starts reading messages from given message lane
+//! of source chain and submits proof-of-message-at-source-chain transactions to the
+//! target chain. Additionally, proofs-of-messages-delivery are sent back from the
+//! target chain to the source chain.
+
+// required for futures::select!
+#![recursion_limit = "1024"]
+#![warn(missing_docs)]
+
+mod metrics;
+
+pub mod message_lane;
+pub mod message_lane_loop;
+
+mod message_race_delivery;
+mod message_race_limits;
+mod message_race_loop;
+mod message_race_receiving;
+mod message_race_strategy;
+
+pub use message_race_delivery::relay_messages_range;
+pub use message_race_receiving::relay_messages_delivery_confirmation;
+pub use metrics::Labeled;

pezbridges/relays/messages/src/message_lane.rs

@@ -0,0 +1,75 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! One-way message lane types. Within single one-way lane we have three 'races' where we try to:
+//!
+//! 1) relay new messages from source to target node;
+//! 2) relay proof-of-delivery from target to source node.
+
+use crate::metrics::Labeled;
+use num_traits::{SaturatingAdd, Zero};
+use relay_utils::{BlockNumberBase, HeaderId};
+use pezsp_arithmetic::traits::AtLeast32BitUnsigned;
+use std::{fmt::Debug, ops::Sub};
+
+/// One-way message lane.
+pub trait MessageLane: 'static + Clone + Send + Sync {
+	/// Name of the messages source.
+	const SOURCE_NAME: &'static str;
+	/// Name of the messages target.
+	const TARGET_NAME: &'static str;
+
+	/// Lane identifier type.
+	type LaneId: Clone + Send + Sync + Labeled;
+
+	/// Messages proof.
+	type MessagesProof: Clone + Debug + Send + Sync;
+	/// Messages receiving proof.
+	type MessagesReceivingProof: Clone + Debug + Send + Sync;
+
+	/// The type of the source chain token balance, that is used to:
+	///
+	/// 1) pay transaction fees;
+	/// 2) pay message delivery and dispatch fee;
+	/// 3) pay relayer rewards.
+	type SourceChainBalance: AtLeast32BitUnsigned
+		+ Clone
+		+ Copy
+		+ Debug
+		+ PartialOrd
+		+ Sub<Output = Self::SourceChainBalance>
+		+ SaturatingAdd
+		+ Zero
+		+ Send
+		+ Sync;
+	/// Number of the source header.
+	type SourceHeaderNumber: BlockNumberBase;
+	/// Hash of the source header.
+	type SourceHeaderHash: Clone + Debug + Default + PartialEq + Send + Sync;
+
+	/// Number of the target header.
+	type TargetHeaderNumber: BlockNumberBase;
+	/// Hash of the target header.
+	type TargetHeaderHash: Clone + Debug + Default + PartialEq + Send + Sync;
+}
+
+/// Source header id within given one-way message lane.
+pub type SourceHeaderIdOf<P> =
+	HeaderId<<P as MessageLane>::SourceHeaderHash, <P as MessageLane>::SourceHeaderNumber>;
+
+/// Target header id within given one-way message lane.
+pub type TargetHeaderIdOf<P> =
+	HeaderId<<P as MessageLane>::TargetHeaderHash, <P as MessageLane>::TargetHeaderNumber>;

pezbridges/relays/messages/src/message_race_limits.rs

@@ -0,0 +1,169 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! enforcement strategy
+
+use num_traits::Zero;
+use std::ops::RangeInclusive;
+
+use bp_messages::{MessageNonce, Weight};
+
+use crate::{
+	message_lane::MessageLane,
+	message_lane_loop::{MessageDetails, MessageDetailsMap},
+	message_race_loop::NoncesRange,
+	message_race_strategy::SourceRangesQueue,
+};
+
+/// Reference data for participating in relay
+pub struct RelayReference<P: MessageLane> {
+	/// Messages size summary
+	pub selected_size: u32,
+
+	/// Index by all ready nonces
+	pub index: usize,
+	/// Current nonce
+	pub nonce: MessageNonce,
+	/// Current nonce details
+	pub details: MessageDetails<P::SourceChainBalance>,
+}
+
+/// Relay reference data
+pub struct RelayMessagesBatchReference<P: MessageLane> {
+	/// Maximal number of relayed messages in single delivery transaction.
+	pub max_messages_in_this_batch: MessageNonce,
+	/// Maximal cumulative dispatch weight of relayed messages in single delivery transaction.
+	pub max_messages_weight_in_single_batch: Weight,
+	/// Maximal cumulative size of relayed messages in single delivery transaction.
+	pub max_messages_size_in_single_batch: u32,
+	/// Best available nonce at the **best** target block. We do not want to deliver nonces
+	/// less than this nonce, even though the block may be retracted.
+	pub best_target_nonce: MessageNonce,
+	/// Source queue.
+	pub nonces_queue: SourceRangesQueue<
+		P::SourceHeaderHash,
+		P::SourceHeaderNumber,
+		MessageDetailsMap<P::SourceChainBalance>,
+	>,
+	/// Range of indices within the `nonces_queue` that are available for selection.
+	pub nonces_queue_range: RangeInclusive<usize>,
+}
+
+/// Limits of the message race transactions.
+#[derive(Clone)]
+pub struct MessageRaceLimits;
+
+impl MessageRaceLimits {
+	pub async fn decide<P: MessageLane>(
+		reference: RelayMessagesBatchReference<P>,
+	) -> Option<RangeInclusive<MessageNonce>> {
+		let mut hard_selected_count = 0;
+
+		let mut selected_weight = Weight::zero();
+		let mut selected_count: MessageNonce = 0;
+
+		let hard_selected_begin_nonce = std::cmp::max(
+			reference.best_target_nonce + 1,
+			reference.nonces_queue[*reference.nonces_queue_range.start()].1.begin(),
+		);
+
+		// relay reference
+		let mut relay_reference = RelayReference::<P> {
+			selected_size: 0,
+
+			index: 0,
+			nonce: 0,
+			details: MessageDetails {
+				dispatch_weight: Weight::zero(),
+				size: 0,
+				reward: P::SourceChainBalance::zero(),
+			},
+		};
+
+		let all_ready_nonces = reference
+			.nonces_queue
+			.range(reference.nonces_queue_range.clone())
+			.flat_map(|(_, ready_nonces)| ready_nonces.iter())
+			.filter(|(nonce, _)| **nonce >= hard_selected_begin_nonce)
+			.enumerate();
+		for (index, (nonce, details)) in all_ready_nonces {
+			relay_reference.index = index;
+			relay_reference.nonce = *nonce;
+			relay_reference.details = *details;
+
+			// Since we (hopefully) have some reserves in `max_messages_weight_in_single_batch`
+			// and `max_messages_size_in_single_batch`, we may still try to submit transaction
+			// with single message if message overflows these limits. The worst case would be if
+			// transaction will be rejected by the target runtime, but at least we have tried.
+
+			// limit messages in the batch by weight
+			let new_selected_weight = match selected_weight.checked_add(&details.dispatch_weight) {
+				Some(new_selected_weight)
+					if new_selected_weight
+						.all_lte(reference.max_messages_weight_in_single_batch) =>
+					new_selected_weight,
+				new_selected_weight if selected_count == 0 => {
+					tracing::warn!(
+						target: "bridge",
+						dispatch_weight=?new_selected_weight,
+						configured_weight=%reference.max_messages_weight_in_single_batch,
+						"Going to submit message delivery transaction with declared dispatch \
+						weight that overflows maximal configured weight"
+					);
+					new_selected_weight.unwrap_or(Weight::MAX)
+				},
+				_ => break,
+			};
+
+			// limit messages in the batch by size
+			let new_selected_size = match relay_reference.selected_size.checked_add(details.size) {
+				Some(new_selected_size)
+					if new_selected_size <= reference.max_messages_size_in_single_batch =>
+					new_selected_size,
+				new_selected_size if selected_count == 0 => {
+					tracing::warn!(
+						target: "bridge",
+						message_size=new_selected_size,
+						configured_size=%reference.max_messages_size_in_single_batch,
+						"Going to submit message delivery transaction with message \
+						size that overflows maximal configured size"
+					);
+					new_selected_size.unwrap_or(u32::MAX)
+				},
+				_ => break,
+			};
+
+			// limit number of messages in the batch
+			let new_selected_count = selected_count + 1;
+			if new_selected_count > reference.max_messages_in_this_batch {
+				break;
+			}
+			relay_reference.selected_size = new_selected_size;
+
+			hard_selected_count = index + 1;
+			selected_weight = new_selected_weight;
+			selected_count = new_selected_count;
+		}
+
+		if hard_selected_count != 0 {
+			let selected_max_nonce =
+				hard_selected_begin_nonce + hard_selected_count as MessageNonce - 1;
+			Some(hard_selected_begin_nonce..=selected_max_nonce)
+		} else {
+			None
+		}
+	}
+}

pezbridges/relays/messages/src/message_race_loop.rs

@@ -0,0 +1,816 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+//! Loop that is serving single race within message lane. This could be
+//! message delivery race, receiving confirmations race or processing
+//! confirmations race.
+//!
+//! The idea of the race is simple - we have `nonce`-s on source and target
+//! nodes. We're trying to prove that the source node has this nonce (and
+//! associated data - like messages, lane state, etc) to the target node by
+//! generating and submitting proof.
+
+use crate::message_lane_loop::{BatchTransaction, ClientState, NoncesSubmitArtifacts};
+
+use async_trait::async_trait;
+use bp_messages::MessageNonce;
+use futures::{
+	future::{FutureExt, TryFutureExt},
+	stream::{FusedStream, StreamExt},
+};
+use relay_utils::{
+	process_future_result, retry_backoff, FailedClient, MaybeConnectionError,
+	TrackedTransactionStatus, TransactionTracker,
+};
+use std::{
+	fmt::Debug,
+	ops::RangeInclusive,
+	time::{Duration, Instant},
+};
+
+/// One of races within lane.
+pub trait MessageRace {
+	/// Header id of the race source.
+	type SourceHeaderId: Debug + Clone + PartialEq + Send + Sync;
+	/// Header id of the race source.
+	type TargetHeaderId: Debug + Clone + PartialEq + Send + Sync;
+
+	/// Message nonce used in the race.
+	type MessageNonce: Debug + Clone;
+	/// Proof that is generated and delivered in this race.
+	type Proof: Debug + Clone + Send + Sync;
+
+	/// Name of the race source.
+	fn source_name() -> String;
+	/// Name of the race target.
+	fn target_name() -> String;
+}
+
+/// State of race source client.
+type SourceClientState<P> =
+	ClientState<<P as MessageRace>::SourceHeaderId, <P as MessageRace>::TargetHeaderId>;
+
+/// State of race target client.
+type TargetClientState<P> =
+	ClientState<<P as MessageRace>::TargetHeaderId, <P as MessageRace>::SourceHeaderId>;
+
+/// Inclusive nonces range.
+pub trait NoncesRange: Debug + Sized {
+	/// Get begin of the range.
+	fn begin(&self) -> MessageNonce;
+	/// Get end of the range.
+	fn end(&self) -> MessageNonce;
+	/// Returns new range with current range nonces that are greater than the passed `nonce`.
+	/// If there are no such nonces, `None` is returned.
+	fn greater_than(self, nonce: MessageNonce) -> Option<Self>;
+}
+
+/// Nonces on the race source client.
+#[derive(Debug, Clone)]
+pub struct SourceClientNonces<NoncesRange> {
+	/// New nonces range known to the client. `New` here means all nonces generated after
+	/// `prev_latest_nonce` passed to the `SourceClient::nonces` method.
+	pub new_nonces: NoncesRange,
+	/// The latest nonce that is confirmed to the bridged client. This nonce only makes
+	/// sense in some races. In other races it is `None`.
+	pub confirmed_nonce: Option<MessageNonce>,
+}
+
+/// Nonces on the race target client.
+#[derive(Debug, Clone)]
+pub struct TargetClientNonces<TargetNoncesData> {
+	/// The latest nonce that is known to the target client.
+	pub latest_nonce: MessageNonce,
+	/// Additional data from target node that may be used by the race.
+	pub nonces_data: TargetNoncesData,
+}
+
+/// One of message lane clients, which is source client for the race.
+#[async_trait]
+pub trait SourceClient<P: MessageRace> {
+	/// Type of error these clients returns.
+	type Error: std::fmt::Debug + MaybeConnectionError;
+	/// Type of nonces range returned by the source client.
+	type NoncesRange: NoncesRange;
+	/// Additional proof parameters required to generate proof.
+	type ProofParameters;
+
+	/// Return nonces that are known to the source client.
+	async fn nonces(
+		&self,
+		at_block: P::SourceHeaderId,
+		prev_latest_nonce: MessageNonce,
+	) -> Result<(P::SourceHeaderId, SourceClientNonces<Self::NoncesRange>), Self::Error>;
+	/// Generate proof for delivering to the target client.
+	async fn generate_proof(
+		&self,
+		at_block: P::SourceHeaderId,
+		nonces: RangeInclusive<MessageNonce>,
+		proof_parameters: Self::ProofParameters,
+	) -> Result<(P::SourceHeaderId, RangeInclusive<MessageNonce>, P::Proof), Self::Error>;
+}
+
+/// One of message lane clients, which is target client for the race.
+#[async_trait]
+pub trait TargetClient<P: MessageRace> {
+	/// Type of error these clients returns.
+	type Error: std::fmt::Debug + MaybeConnectionError;
+	/// Type of the additional data from the target client, used by the race.
+	type TargetNoncesData: std::fmt::Debug;
+	/// Type of batch transaction that submits finality and proof to the target node.
+	type BatchTransaction: BatchTransaction<P::SourceHeaderId> + Clone;
+	/// Transaction tracker to track submitted transactions.
+	type TransactionTracker: TransactionTracker<HeaderId = P::TargetHeaderId>;
+
+	/// Ask headers relay to relay finalized headers up to (and including) given header
+	/// from race source to race target.
+	///
+	/// The client may return `Some(_)`, which means that nothing has happened yet and
+	/// the caller must generate and append proof to the batch transaction
+	/// to actually send it (along with required header) to the node.
+	///
+	/// If function has returned `None`, it means that the caller now must wait for the
+	/// appearance of the required header `id` at the target client.
+	async fn require_source_header(
+		&self,
+		id: P::SourceHeaderId,
+	) -> Result<Option<Self::BatchTransaction>, Self::Error>;
+
+	/// Return nonces that are known to the target client.
+	async fn nonces(
+		&self,
+		at_block: P::TargetHeaderId,
+		update_metrics: bool,
+	) -> Result<(P::TargetHeaderId, TargetClientNonces<Self::TargetNoncesData>), Self::Error>;
+	/// Submit proof to the target client.
+	async fn submit_proof(
+		&self,
+		maybe_batch_tx: Option<Self::BatchTransaction>,
+		generated_at_block: P::SourceHeaderId,
+		nonces: RangeInclusive<MessageNonce>,
+		proof: P::Proof,
+	) -> Result<NoncesSubmitArtifacts<Self::TransactionTracker>, Self::Error>;
+}
+
+/// Race strategy.
+#[async_trait]
+pub trait RaceStrategy<SourceHeaderId, TargetHeaderId, Proof>: Debug {
+	/// Type of nonces range expected from the source client.
+	type SourceNoncesRange: NoncesRange;
+	/// Additional proof parameters required to generate proof.
+	type ProofParameters;
+	/// Additional data expected from the target client.
+	type TargetNoncesData;
+
+	/// Return id of source header that is required to be on target to continue synchronization.
+	async fn required_source_header_at_target<RS: RaceState<SourceHeaderId, TargetHeaderId>>(
+		&self,
+		race_state: RS,
+	) -> Option<SourceHeaderId>;
+	/// Return the best nonce at source node.
+	///
+	/// `Some` is returned only if we are sure that the value is greater or equal
+	/// than the result of `best_at_target`.
+	fn best_at_source(&self) -> Option<MessageNonce>;
+	/// Return the best nonce at target node.
+	///
+	/// May return `None` if value is yet unknown.
+	fn best_at_target(&self) -> Option<MessageNonce>;
+
+	/// Called when nonces are updated at source node of the race.
+	fn source_nonces_updated(
+		&mut self,
+		at_block: SourceHeaderId,
+		nonces: SourceClientNonces<Self::SourceNoncesRange>,
+	);
+	/// Called when we want to wait until next `best_target_nonces_updated` before selecting
+	/// any nonces for delivery.
+	fn reset_best_target_nonces(&mut self);
+	/// Called when best nonces are updated at target node of the race.
+	fn best_target_nonces_updated<RS: RaceState<SourceHeaderId, TargetHeaderId>>(
+		&mut self,
+		nonces: TargetClientNonces<Self::TargetNoncesData>,
+		race_state: &mut RS,
+	);
+	/// Called when finalized nonces are updated at target node of the race.
+	fn finalized_target_nonces_updated<RS: RaceState<SourceHeaderId, TargetHeaderId>>(
+		&mut self,
+		nonces: TargetClientNonces<Self::TargetNoncesData>,
+		race_state: &mut RS,
+	);
+	/// Should return `Some(nonces)` if we need to deliver proof of `nonces` (and associated
+	/// data) from source to target node.
+	/// Additionally, parameters required to generate proof are returned.
+	async fn select_nonces_to_deliver<RS: RaceState<SourceHeaderId, TargetHeaderId>>(
+		&self,
+		race_state: RS,
+	) -> Option<(RangeInclusive<MessageNonce>, Self::ProofParameters)>;
+}
+
+/// State of the race.
+pub trait RaceState<SourceHeaderId, TargetHeaderId>: Clone + Send + Sync {
+	/// Set best finalized source header id at the best block on the target
+	/// client (at the `best_finalized_source_header_id_at_best_target`).
+	fn set_best_finalized_source_header_id_at_best_target(&mut self, id: SourceHeaderId);
+
+	/// Best finalized source header id at the best block on the target
+	/// client (at the `best_finalized_source_header_id_at_best_target`).
+	fn best_finalized_source_header_id_at_best_target(&self) -> Option<SourceHeaderId>;
+
+	/// Returns `true` if we have selected nonces to submit to the target node.
+	fn nonces_to_submit(&self) -> Option<RangeInclusive<MessageNonce>>;
+	/// Reset our nonces selection.
+	fn reset_nonces_to_submit(&mut self);
+
+	/// Returns `true` if we have submitted some nonces to the target node and are
+	/// waiting for them to appear there.
+	fn nonces_submitted(&self) -> Option<RangeInclusive<MessageNonce>>;
+	/// Reset our nonces submission.
+	fn reset_nonces_submitted(&mut self);
+}
+
+/// State of the race and prepared batch transaction (if available).
+#[derive(Debug, Clone)]
+pub(crate) struct RaceStateImpl<SourceHeaderId, TargetHeaderId, Proof, BatchTx> {
+	/// Best finalized source header id at the source client.
+	pub best_finalized_source_header_id_at_source: Option<SourceHeaderId>,
+	/// Best finalized source header id at the best block on the target
+	/// client (at the `best_finalized_source_header_id_at_best_target`).
+	pub best_finalized_source_header_id_at_best_target: Option<SourceHeaderId>,
+	/// The best header id at the target client.
+	pub best_target_header_id: Option<TargetHeaderId>,
+	/// Best finalized header id at the target client.
+	pub best_finalized_target_header_id: Option<TargetHeaderId>,
+	/// Range of nonces that we have selected to submit.
+	pub nonces_to_submit: Option<(SourceHeaderId, RangeInclusive<MessageNonce>, Proof)>,
+	/// Batch transaction ready to include and deliver selected `nonces_to_submit` from the
+	/// `state`.
+	pub nonces_to_submit_batch: Option<BatchTx>,
+	/// Range of nonces that is currently submitted.
+	pub nonces_submitted: Option<RangeInclusive<MessageNonce>>,
+}
+
+impl<SourceHeaderId, TargetHeaderId, Proof, BatchTx> Default
+	for RaceStateImpl<SourceHeaderId, TargetHeaderId, Proof, BatchTx>
+{
+	fn default() -> Self {
+		RaceStateImpl {
+			best_finalized_source_header_id_at_source: None,
+			best_finalized_source_header_id_at_best_target: None,
+			best_target_header_id: None,
+			best_finalized_target_header_id: None,
+			nonces_to_submit: None,
+			nonces_to_submit_batch: None,
+			nonces_submitted: None,
+		}
+	}
+}
+
+impl<SourceHeaderId, TargetHeaderId, Proof, BatchTx> RaceState<SourceHeaderId, TargetHeaderId>
+	for RaceStateImpl<SourceHeaderId, TargetHeaderId, Proof, BatchTx>
+where
+	SourceHeaderId: Clone + Send + Sync,
+	TargetHeaderId: Clone + Send + Sync,
+	Proof: Clone + Send + Sync,
+	BatchTx: Clone + Send + Sync,
+{
+	fn set_best_finalized_source_header_id_at_best_target(&mut self, id: SourceHeaderId) {
+		self.best_finalized_source_header_id_at_best_target = Some(id);
+	}
+
+	fn best_finalized_source_header_id_at_best_target(&self) -> Option<SourceHeaderId> {
+		self.best_finalized_source_header_id_at_best_target.clone()
+	}
+
+	fn nonces_to_submit(&self) -> Option<RangeInclusive<MessageNonce>> {
+		self.nonces_to_submit.clone().map(|(_, nonces, _)| nonces)
+	}
+
+	fn reset_nonces_to_submit(&mut self) {
+		self.nonces_to_submit = None;
+		self.nonces_to_submit_batch = None;
+	}
+
+	fn nonces_submitted(&self) -> Option<RangeInclusive<MessageNonce>> {
+		self.nonces_submitted.clone()
+	}
+
+	fn reset_nonces_submitted(&mut self) {
+		self.nonces_submitted = None;
+	}
+}
+
+/// Run race loop until connection with target or source node is lost.
+pub async fn run<P: MessageRace, SC: SourceClient<P>, TC: TargetClient<P>>(
+	race_source: SC,
+	race_source_updated: impl FusedStream<Item = SourceClientState<P>>,
+	race_target: TC,
+	race_target_updated: impl FusedStream<Item = TargetClientState<P>>,
+	mut strategy: impl RaceStrategy<
+		P::SourceHeaderId,
+		P::TargetHeaderId,
+		P::Proof,
+		SourceNoncesRange = SC::NoncesRange,
+		ProofParameters = SC::ProofParameters,
+		TargetNoncesData = TC::TargetNoncesData,
+	>,
+) -> Result<(), FailedClient> {
+	let mut progress_context = Instant::now();
+	let mut race_state = RaceStateImpl::default();
+
+	let mut source_retry_backoff = retry_backoff();
+	let mut source_client_is_online = true;
+	let mut source_nonces_required = false;
+	let mut source_required_header = None;
+	let source_nonces = futures::future::Fuse::terminated();
+	let source_generate_proof = futures::future::Fuse::terminated();
+	let source_go_offline_future = futures::future::Fuse::terminated();
+
+	let mut target_retry_backoff = retry_backoff();
+	let mut target_client_is_online = true;
+	let mut target_best_nonces_required = false;
+	let mut target_finalized_nonces_required = false;
+	let mut target_batch_transaction = None;
+	let target_require_source_header = futures::future::Fuse::terminated();
+	let target_best_nonces = futures::future::Fuse::terminated();
+	let target_finalized_nonces = futures::future::Fuse::terminated();
+	let target_submit_proof = futures::future::Fuse::terminated();
+	let target_tx_tracker = futures::future::Fuse::terminated();
+	let target_go_offline_future = futures::future::Fuse::terminated();
+
+	futures::pin_mut!(
+		race_source_updated,
+		source_nonces,
+		source_generate_proof,
+		source_go_offline_future,
+		race_target_updated,
+		target_require_source_header,
+		target_best_nonces,
+		target_finalized_nonces,
+		target_submit_proof,
+		target_tx_tracker,
+		target_go_offline_future,
+	);
+
+	loop {
+		futures::select! {
+			// when headers ids are updated
+			source_state = race_source_updated.next() => {
+				if let Some(source_state) = source_state {
+					let is_source_state_updated = race_state.best_finalized_source_header_id_at_source.as_ref()
+						!= Some(&source_state.best_finalized_self);
+					if is_source_state_updated {
+						source_nonces_required = true;
+						race_state.best_finalized_source_header_id_at_source
+							= Some(source_state.best_finalized_self);
+					}
+				}
+			},
+			target_state = race_target_updated.next() => {
+				if let Some(target_state) = target_state {
+					let is_target_best_state_updated = race_state.best_target_header_id.as_ref()
+						!= Some(&target_state.best_self);
+
+					if is_target_best_state_updated {
+						target_best_nonces_required = true;
+						race_state.best_target_header_id = Some(target_state.best_self);
+						race_state.best_finalized_source_header_id_at_best_target
+							= target_state.best_finalized_peer_at_best_self;
+					}
+
+					let is_target_finalized_state_updated = race_state.best_finalized_target_header_id.as_ref()
+						!= Some(&target_state.best_finalized_self);
+					if  is_target_finalized_state_updated {
+						target_finalized_nonces_required = true;
+						race_state.best_finalized_target_header_id = Some(target_state.best_finalized_self);
+					}
+				}
+			},
+
+			// when nonces are updated
+			nonces = source_nonces => {
+				source_nonces_required = false;
+
+				source_client_is_online = process_future_result(
+					nonces,
+					&mut source_retry_backoff,
+					|(at_block, nonces)| {
+						tracing::debug!(
+							target: "bridge",
+							source=%P::source_name(),
+							?nonces,
+							"Received nonces"
+						);
+
+						strategy.source_nonces_updated(at_block, nonces);
+					},
+					&mut source_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving nonces from {}", P::source_name()),
+				).fail_if_connection_error(FailedClient::Source)?;
+
+				// ask for more headers if we have nonces to deliver and required headers are missing
+				source_required_header = strategy
+					.required_source_header_at_target(race_state.clone())
+					.await;
+			},
+			nonces = target_best_nonces => {
+				target_best_nonces_required = false;
+
+				target_client_is_online = process_future_result(
+					nonces,
+					&mut target_retry_backoff,
+					|(_, nonces)| {
+						tracing::debug!(
+							target: "bridge",
+							target=%P::target_name(),
+							?nonces,
+							"Received best nonces"
+						);
+
+						strategy.best_target_nonces_updated(nonces, &mut race_state);
+					},
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving best nonces from {}", P::target_name()),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+			nonces = target_finalized_nonces => {
+				target_finalized_nonces_required = false;
+
+				target_client_is_online = process_future_result(
+					nonces,
+					&mut target_retry_backoff,
+					|(_, nonces)| {
+						tracing::debug!(
+							target: "bridge",
+							target=%P::target_name(),
+							?nonces,
+							"Received finalized nonces"
+						);
+
+						strategy.finalized_target_nonces_updated(nonces, &mut race_state);
+					},
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error retrieving finalized nonces from {}", P::target_name()),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+
+			// proof generation and submission
+			maybe_batch_transaction = target_require_source_header => {
+				source_required_header = None;
+
+				target_client_is_online = process_future_result(
+					maybe_batch_transaction,
+					&mut target_retry_backoff,
+					|maybe_batch_transaction: Option<TC::BatchTransaction>| {
+						tracing::debug!(
+							target: "bridge",
+							target=%P::target_name(),
+							source=%P::source_name(),
+							batch_tx=%maybe_batch_transaction
+								.as_ref()
+								.map(|bt| format!("yes ({:?})", bt.required_header_id()))
+								.unwrap_or_else(|| "no".into()),
+							"Target client has been asked for more headers."
+						);
+
+						target_batch_transaction = maybe_batch_transaction;
+					},
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error asking for source headers at {}", P::target_name()),
+				).fail_if_connection_error(FailedClient::Target)?;
+			},
+			proof = source_generate_proof => {
+				source_client_is_online = process_future_result(
+					proof,
+					&mut source_retry_backoff,
+					|(at_block, nonces_range, proof, batch_transaction)| {
+						tracing::debug!(
+							target: "bridge",
+							source=%P::source_name(),
+							?nonces_range,
+							"Received proof"
+						);
+
+						race_state.nonces_to_submit = Some((at_block, nonces_range, proof));
+						race_state.nonces_to_submit_batch = batch_transaction;
+					},
+					&mut source_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error generating proof at {}", P::source_name()),
+				).fail_if_error(FailedClient::Source).map(|_| true)?;
+			},
+			proof_submit_result = target_submit_proof => {
+				target_client_is_online = process_future_result(
+					proof_submit_result,
+					&mut target_retry_backoff,
+					|artifacts: NoncesSubmitArtifacts<TC::TransactionTracker>| {
+						tracing::debug!(
+							target: "bridge",
+							target=%P::target_name(),
+							nonces=?artifacts.nonces,
+							"Successfully submitted proof"
+						);
+
+						race_state.nonces_submitted = Some(artifacts.nonces);
+						target_tx_tracker.set(artifacts.tx_tracker.wait().fuse());
+					},
+					&mut target_go_offline_future,
+					async_std::task::sleep,
+					|| format!("Error submitting proof {}", P::target_name()),
+				).fail_if_connection_error(FailedClient::Target)?;
+
+				// in any case - we don't need to retry submitting the same nonces again until
+				// we read nonces from the target client
+				race_state.reset_nonces_to_submit();
+				// if we have failed to submit transaction AND that is not the connection issue,
+				// then we need to read best target nonces before selecting nonces again
+				if !target_client_is_online {
+					strategy.reset_best_target_nonces();
+				}
+			},
+			target_transaction_status = target_tx_tracker => {
+				match (target_transaction_status, race_state.nonces_submitted.as_ref()) {
+					(TrackedTransactionStatus::Finalized(at_block), Some(nonces_submitted)) => {
+						// our transaction has been mined, but was it successful or not? let's check the best
+						// nonce at the target node.
+						let _ = race_target.nonces(at_block, false)
+							.await
+							.map_err(|e| format!("failed to read nonces from target node: {e:?}"))
+							.and_then(|(_, nonces_at_target)| {
+								if nonces_at_target.latest_nonce < *nonces_submitted.end() {
+									Err(format!(
+										"best nonce at target after tx is {:?} and we've submitted {:?}",
+										nonces_at_target.latest_nonce,
+										nonces_submitted.end(),
+									))
+								} else {
+									Ok(())
+								}
+							})
+							.map_err(|e| {
+								tracing::error!(
+									target: "bridge",
+									error=%e,
+									source=%P::source_name(),
+									target=%P::target_name(),
+									"Source -> target race transaction failed"
+								);
+
+								race_state.reset_nonces_submitted();
+							});
+					},
+					(TrackedTransactionStatus::Lost, _) => {
+						tracing::warn!(
+							target: "bridge",
+							source=%P::source_name(),
+							target=%P::target_name(),
+							state=?race_state,
+							?strategy,
+							"Source -> target race transaction has been lost."
+						);
+
+						race_state.reset_nonces_submitted();
+					},
+					_ => (),
+				}
+			},
+
+			// when we're ready to retry request
+			_ = source_go_offline_future => {
+				source_client_is_online = true;
+			},
+			_ = target_go_offline_future => {
+				target_client_is_online = true;
+			},
+		}
+
+		progress_context = print_race_progress::<P, _>(progress_context, &strategy);
+
+		if source_client_is_online {
+			source_client_is_online = false;
+
+			// if we've started to submit batch transaction, let's prioritize it
+			//
+			// we're using `take` here, because we don't need batch transaction (i.e. some
+			// underlying finality proof) anymore for our future calls - we were unable to
+			// use it for our current state, so why would we need to keep an obsolete proof
+			// for the future?
+			let target_batch_transaction = target_batch_transaction.take();
+			let expected_race_state =
+				if let Some(ref target_batch_transaction) = target_batch_transaction {
+					// when selecting nonces for the batch transaction, we assume that the required
+					// source header is already at the target chain
+					let required_source_header_at_target =
+						target_batch_transaction.required_header_id();
+					let mut expected_race_state = race_state.clone();
+					expected_race_state.best_finalized_source_header_id_at_best_target =
+						Some(required_source_header_at_target);
+					expected_race_state
+				} else {
+					race_state.clone()
+				};
+
+			let nonces_to_deliver = select_nonces_to_deliver(expected_race_state, &strategy).await;
+			let best_at_source = strategy.best_at_source();
+
+			if let Some((at_block, nonces_range, proof_parameters)) = nonces_to_deliver {
+				tracing::debug!(
+					target: "bridge",
+					source=P::source_name(),
+					?nonces_range,
+					?at_block,
+					"Asking to prove"
+				);
+
+				source_generate_proof.set(
+					race_source
+						.generate_proof(at_block, nonces_range, proof_parameters)
+						.and_then(|(at_source_block, nonces, proof)| async {
+							Ok((at_source_block, nonces, proof, target_batch_transaction))
+						})
+						.fuse(),
+				);
+			} else if let (true, Some(best_at_source)) = (source_nonces_required, best_at_source) {
+				tracing::debug!(target: "bridge", source=%P::source_name(), "Asking about message nonces");
+				let at_block = race_state
+					.best_finalized_source_header_id_at_source
+					.as_ref()
+					.expect(
+						"source_nonces_required is only true when \
+						best_finalized_source_header_id_at_source is Some; qed",
+					)
+					.clone();
+				source_nonces.set(race_source.nonces(at_block, best_at_source).fuse());
+			} else {
+				source_client_is_online = true;
+			}
+		}
+
+		if target_client_is_online {
+			target_client_is_online = false;
+
+			if let Some((at_block, nonces_range, proof)) = race_state.nonces_to_submit.as_ref() {
+				tracing::debug!(
+					target: "bridge",
+					target=%P::target_name(),
+					?nonces_range,
+					"Going to submit proof of messages in range to node{}",
+					race_state.nonces_to_submit_batch.as_ref().map(|tx| format!(
+						". This transaction is batched with sending the proof for header {:?}.",
+						tx.required_header_id())
+					).unwrap_or_default(),
+				);
+
+				target_submit_proof.set(
+					race_target
+						.submit_proof(
+							race_state.nonces_to_submit_batch.clone(),
+							at_block.clone(),
+							nonces_range.clone(),
+							proof.clone(),
+						)
+						.fuse(),
+				);
+			} else if let Some(source_required_header) = source_required_header.clone() {
+				tracing::debug!(
+					target: "bridge",
+					source=%P::source_name(),
+					target=%P::target_name(),
+					?source_required_header,
+					"Going to require header"
+				);
+				target_require_source_header
+					.set(race_target.require_source_header(source_required_header).fuse());
+			} else if target_best_nonces_required {
+				tracing::debug!(target: "bridge", target=%P::target_name(), "Asking about best message nonces");
+				let at_block = race_state
+					.best_target_header_id
+					.as_ref()
+					.expect("target_best_nonces_required is only true when best_target_header_id is Some; qed")
+					.clone();
+				target_best_nonces.set(race_target.nonces(at_block, false).fuse());
+			} else if target_finalized_nonces_required {
+				tracing::debug!(target: "bridge", target=%P::target_name(), "Asking about finalized message nonces");
+				let at_block = race_state
+					.best_finalized_target_header_id
+					.as_ref()
+					.expect(
+						"target_finalized_nonces_required is only true when \
+						best_finalized_target_header_id is Some; qed",
+					)
+					.clone();
+				target_finalized_nonces.set(race_target.nonces(at_block, true).fuse());
+			} else {
+				target_client_is_online = true;
+			}
+		}
+	}
+}
+
+/// Print race progress.
+fn print_race_progress<P, S>(prev_time: Instant, strategy: &S) -> Instant
+where
+	P: MessageRace,
+	S: RaceStrategy<P::SourceHeaderId, P::TargetHeaderId, P::Proof>,
+{
+	let now_time = Instant::now();
+
+	let need_update = now_time.saturating_duration_since(prev_time) > Duration::from_secs(10);
+	if !need_update {
+		return prev_time;
+	}
+
+	let now_best_nonce_at_source = strategy.best_at_source();
+	let now_best_nonce_at_target = strategy.best_at_target();
+	tracing::info!(
+		target: "bridge",
+		source=%P::source_name(),
+		target=%P::target_name(),
+		?now_best_nonce_at_target,
+		?now_best_nonce_at_source,
+		"Synced nonces in source -> target race"
+
+	);
+	now_time
+}
+
+async fn select_nonces_to_deliver<SourceHeaderId, TargetHeaderId, Proof, Strategy>(
+	race_state: impl RaceState<SourceHeaderId, TargetHeaderId>,
+	strategy: &Strategy,
+) -> Option<(SourceHeaderId, RangeInclusive<MessageNonce>, Strategy::ProofParameters)>
+where
+	SourceHeaderId: Clone,
+	Strategy: RaceStrategy<SourceHeaderId, TargetHeaderId, Proof>,
+{
+	let best_finalized_source_header_id_at_best_target =
+		race_state.best_finalized_source_header_id_at_best_target()?;
+	strategy
+		.select_nonces_to_deliver(race_state)
+		.await
+		.map(|(nonces_range, proof_parameters)| {
+			(best_finalized_source_header_id_at_best_target, nonces_range, proof_parameters)
+		})
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::message_race_strategy::BasicStrategy;
+	use relay_utils::HeaderId;
+
+	#[async_std::test]
+	async fn proof_is_generated_at_best_block_known_to_target_node() {
+		const GENERATED_AT: u64 = 6;
+		const BEST_AT_SOURCE: u64 = 10;
+		const BEST_AT_TARGET: u64 = 8;
+
+		// target node only knows about source' BEST_AT_TARGET block
+		// source node has BEST_AT_SOURCE > BEST_AT_TARGET block
+		let mut race_state = RaceStateImpl::<_, _, (), ()> {
+			best_finalized_source_header_id_at_source: Some(HeaderId(
+				BEST_AT_SOURCE,
+				BEST_AT_SOURCE,
+			)),
+			best_finalized_source_header_id_at_best_target: Some(HeaderId(
+				BEST_AT_TARGET,
+				BEST_AT_TARGET,
+			)),
+			best_target_header_id: Some(HeaderId(0, 0)),
+			best_finalized_target_header_id: Some(HeaderId(0, 0)),
+			nonces_to_submit: None,
+			nonces_to_submit_batch: None,
+			nonces_submitted: None,
+		};
+
+		// we have some nonces to deliver and they're generated at GENERATED_AT < BEST_AT_SOURCE
+		let mut strategy = BasicStrategy::<_, _, _, _, _, ()>::new();
+		strategy.source_nonces_updated(
+			HeaderId(GENERATED_AT, GENERATED_AT),
+			SourceClientNonces { new_nonces: 0..=10, confirmed_nonce: None },
+		);
+		strategy.best_target_nonces_updated(
+			TargetClientNonces { latest_nonce: 5u64, nonces_data: () },
+			&mut race_state,
+		);
+
+		// the proof will be generated on source, but using BEST_AT_TARGET block
+		assert_eq!(
+			select_nonces_to_deliver(race_state, &strategy).await,
+			Some((HeaderId(BEST_AT_TARGET, BEST_AT_TARGET), 6..=10, (),))
+		);
+	}
+}

pezbridges/relays/messages/src/message_race_receiving.rs

@@ -0,0 +1,272 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+//! Message receiving race delivers proof-of-messages-delivery from "lane.target" to "lane.source".
+
+use crate::{
+	message_lane::{MessageLane, SourceHeaderIdOf, TargetHeaderIdOf},
+	message_lane_loop::{
+		NoncesSubmitArtifacts, SourceClient as MessageLaneSourceClient, SourceClientState,
+		TargetClient as MessageLaneTargetClient, TargetClientState,
+	},
+	message_race_loop::{
+		MessageRace, NoncesRange, SourceClient, SourceClientNonces, TargetClient,
+		TargetClientNonces,
+	},
+	message_race_strategy::BasicStrategy,
+	metrics::MessageLaneLoopMetrics,
+};
+
+use async_trait::async_trait;
+use bp_messages::MessageNonce;
+use futures::stream::FusedStream;
+use relay_utils::{FailedClient, TrackedTransactionStatus, TransactionTracker};
+use std::{marker::PhantomData, ops::RangeInclusive};
+
+/// Message receiving confirmations delivery strategy.
+type ReceivingConfirmationsBasicStrategy<P> = BasicStrategy<
+	<P as MessageLane>::TargetHeaderNumber,
+	<P as MessageLane>::TargetHeaderHash,
+	<P as MessageLane>::SourceHeaderNumber,
+	<P as MessageLane>::SourceHeaderHash,
+	RangeInclusive<MessageNonce>,
+	<P as MessageLane>::MessagesReceivingProof,
+>;
+
+/// Run receiving confirmations race.
+pub async fn run<P: MessageLane>(
+	source_client: impl MessageLaneSourceClient<P>,
+	source_state_updates: impl FusedStream<Item = SourceClientState<P>>,
+	target_client: impl MessageLaneTargetClient<P>,
+	target_state_updates: impl FusedStream<Item = TargetClientState<P>>,
+	metrics_msg: Option<MessageLaneLoopMetrics>,
+) -> Result<(), FailedClient> {
+	crate::message_race_loop::run(
+		ReceivingConfirmationsRaceSource {
+			client: target_client,
+			metrics_msg: metrics_msg.clone(),
+			_phantom: Default::default(),
+		},
+		target_state_updates,
+		ReceivingConfirmationsRaceTarget {
+			client: source_client,
+			metrics_msg,
+			_phantom: Default::default(),
+		},
+		source_state_updates,
+		ReceivingConfirmationsBasicStrategy::<P>::new(),
+	)
+	.await
+}
+
+/// Relay messages delivery confirmation.
+pub async fn relay_messages_delivery_confirmation<P: MessageLane>(
+	source_client: impl MessageLaneSourceClient<P>,
+	target_client: impl MessageLaneTargetClient<P>,
+	at: TargetHeaderIdOf<P>,
+) -> Result<(), ()> {
+	// prepare messages delivery proof
+	let (at, proof) = target_client.prove_messages_receiving(at.clone()).await.map_err(|e| {
+		tracing::error!(
+			target: "bridge",
+			error=?e,
+			?at,
+			"Failed to generate messages delivery proof",
+		);
+	})?;
+	// submit messages delivery proof to the source node
+	let tx_tracker =
+		source_client
+			.submit_messages_receiving_proof(None, at, proof)
+			.await
+			.map_err(|e| {
+				tracing::error!(
+					target: "bridge",
+					error=?e,
+					"Failed to submit messages delivery proof"
+				);
+			})?;
+
+	match tx_tracker.wait().await {
+		TrackedTransactionStatus::Finalized(_) => Ok(()),
+		TrackedTransactionStatus::Lost => {
+			tracing::error!(target: "bridge", "Transaction with messages delivery proof is considered lost");
+			Err(())
+		},
+	}
+}
+
+/// Messages receiving confirmations race.
+struct ReceivingConfirmationsRace<P>(std::marker::PhantomData<P>);
+
+impl<P: MessageLane> MessageRace for ReceivingConfirmationsRace<P> {
+	type SourceHeaderId = TargetHeaderIdOf<P>;
+	type TargetHeaderId = SourceHeaderIdOf<P>;
+
+	type MessageNonce = MessageNonce;
+	type Proof = P::MessagesReceivingProof;
+
+	fn source_name() -> String {
+		format!("{}::ReceivingConfirmationsDelivery", P::TARGET_NAME)
+	}
+
+	fn target_name() -> String {
+		format!("{}::ReceivingConfirmationsDelivery", P::SOURCE_NAME)
+	}
+}
+
+/// Message receiving confirmations race source, which is a target of the lane.
+struct ReceivingConfirmationsRaceSource<P: MessageLane, C> {
+	client: C,
+	metrics_msg: Option<MessageLaneLoopMetrics>,
+	_phantom: PhantomData<P>,
+}
+
+#[async_trait]
+impl<P, C> SourceClient<ReceivingConfirmationsRace<P>> for ReceivingConfirmationsRaceSource<P, C>
+where
+	P: MessageLane,
+	C: MessageLaneTargetClient<P>,
+{
+	type Error = C::Error;
+	type NoncesRange = RangeInclusive<MessageNonce>;
+	type ProofParameters = ();
+
+	async fn nonces(
+		&self,
+		at_block: TargetHeaderIdOf<P>,
+		prev_latest_nonce: MessageNonce,
+	) -> Result<(TargetHeaderIdOf<P>, SourceClientNonces<Self::NoncesRange>), Self::Error> {
+		let (at_block, latest_received_nonce) = self.client.latest_received_nonce(at_block).await?;
+		if let Some(metrics_msg) = self.metrics_msg.as_ref() {
+			metrics_msg.update_target_latest_received_nonce(latest_received_nonce);
+		}
+		Ok((
+			at_block,
+			SourceClientNonces {
+				new_nonces: prev_latest_nonce + 1..=latest_received_nonce,
+				confirmed_nonce: None,
+			},
+		))
+	}
+
+	#[allow(clippy::unit_arg)]
+	async fn generate_proof(
+		&self,
+		at_block: TargetHeaderIdOf<P>,
+		nonces: RangeInclusive<MessageNonce>,
+		_proof_parameters: Self::ProofParameters,
+	) -> Result<
+		(TargetHeaderIdOf<P>, RangeInclusive<MessageNonce>, P::MessagesReceivingProof),
+		Self::Error,
+	> {
+		self.client
+			.prove_messages_receiving(at_block)
+			.await
+			.map(|(at_block, proof)| (at_block, nonces, proof))
+	}
+}
+
+/// Message receiving confirmations race target, which is a source of the lane.
+struct ReceivingConfirmationsRaceTarget<P: MessageLane, C> {
+	client: C,
+	metrics_msg: Option<MessageLaneLoopMetrics>,
+	_phantom: PhantomData<P>,
+}
+
+#[async_trait]
+impl<P, C> TargetClient<ReceivingConfirmationsRace<P>> for ReceivingConfirmationsRaceTarget<P, C>
+where
+	P: MessageLane,
+	C: MessageLaneSourceClient<P>,
+{
+	type Error = C::Error;
+	type TargetNoncesData = ();
+	type BatchTransaction = C::BatchTransaction;
+	type TransactionTracker = C::TransactionTracker;
+
+	async fn require_source_header(
+		&self,
+		id: TargetHeaderIdOf<P>,
+	) -> Result<Option<C::BatchTransaction>, Self::Error> {
+		self.client.require_target_header_on_source(id).await
+	}
+
+	async fn nonces(
+		&self,
+		at_block: SourceHeaderIdOf<P>,
+		update_metrics: bool,
+	) -> Result<(SourceHeaderIdOf<P>, TargetClientNonces<()>), Self::Error> {
+		let (at_block, latest_confirmed_nonce) =
+			self.client.latest_confirmed_received_nonce(at_block).await?;
+		if update_metrics {
+			if let Some(metrics_msg) = self.metrics_msg.as_ref() {
+				metrics_msg.update_source_latest_confirmed_nonce(latest_confirmed_nonce);
+			}
+		}
+		Ok((at_block, TargetClientNonces { latest_nonce: latest_confirmed_nonce, nonces_data: () }))
+	}
+
+	async fn submit_proof(
+		&self,
+		maybe_batch_tx: Option<Self::BatchTransaction>,
+		generated_at_block: TargetHeaderIdOf<P>,
+		nonces: RangeInclusive<MessageNonce>,
+		proof: P::MessagesReceivingProof,
+	) -> Result<NoncesSubmitArtifacts<Self::TransactionTracker>, Self::Error> {
+		let tx_tracker = self
+			.client
+			.submit_messages_receiving_proof(maybe_batch_tx, generated_at_block, proof)
+			.await?;
+		Ok(NoncesSubmitArtifacts { nonces, tx_tracker })
+	}
+}
+
+impl NoncesRange for RangeInclusive<MessageNonce> {
+	fn begin(&self) -> MessageNonce {
+		*RangeInclusive::<MessageNonce>::start(self)
+	}
+
+	fn end(&self) -> MessageNonce {
+		*RangeInclusive::<MessageNonce>::end(self)
+	}
+
+	fn greater_than(self, nonce: MessageNonce) -> Option<Self> {
+		let next_nonce = nonce + 1;
+		let end = *self.end();
+		if next_nonce > end {
+			None
+		} else {
+			Some(std::cmp::max(self.begin(), next_nonce)..=end)
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn range_inclusive_works_as_nonces_range() {
+		let range = 20..=30;
+
+		assert_eq!(NoncesRange::begin(&range), 20);
+		assert_eq!(NoncesRange::end(&range), 30);
+		assert_eq!(range.clone().greater_than(10), Some(20..=30));
+		assert_eq!(range.clone().greater_than(19), Some(20..=30));
+		assert_eq!(range.clone().greater_than(20), Some(21..=30));
+		assert_eq!(range.clone().greater_than(25), Some(26..=30));
+		assert_eq!(range.clone().greater_than(29), Some(30..=30));
+		assert_eq!(range.greater_than(30), None);
+	}
+}

pezbridges/relays/messages/src/message_race_strategy.rs

@@ -0,0 +1,618 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+//! Basic delivery strategy. The strategy selects nonces if:
+//!
+//! 1) there are more nonces on the source side than on the target side;
+//! 2) new nonces may be proved to target node (i.e. they have appeared at the block, which is known
+//!    to the target node).
+
+use crate::message_race_loop::{
+	NoncesRange, RaceState, RaceStrategy, SourceClientNonces, TargetClientNonces,
+};
+
+use async_trait::async_trait;
+use bp_messages::MessageNonce;
+use relay_utils::HeaderId;
+use std::{collections::VecDeque, fmt::Debug, marker::PhantomData, ops::RangeInclusive};
+
+/// Queue of nonces known to the source node.
+pub type SourceRangesQueue<SourceHeaderHash, SourceHeaderNumber, SourceNoncesRange> =
+	VecDeque<(HeaderId<SourceHeaderHash, SourceHeaderNumber>, SourceNoncesRange)>;
+
+/// Nonces delivery strategy.
+#[derive(Debug)]
+pub struct BasicStrategy<
+	SourceHeaderNumber,
+	SourceHeaderHash,
+	TargetHeaderNumber,
+	TargetHeaderHash,
+	SourceNoncesRange,
+	Proof,
+> {
+	/// All queued nonces.
+	///
+	/// The queue may contain already delivered nonces. We only remove entries from this
+	/// queue after corresponding nonces are finalized by the target chain.
+	source_queue: SourceRangesQueue<SourceHeaderHash, SourceHeaderNumber, SourceNoncesRange>,
+	/// The best nonce known to target node at its best block. `None` if it has not been received
+	/// yet.
+	best_target_nonce: Option<MessageNonce>,
+	/// Unused generic types dump.
+	_phantom: PhantomData<(TargetHeaderNumber, TargetHeaderHash, Proof)>,
+}
+
+impl<
+		SourceHeaderNumber,
+		SourceHeaderHash,
+		TargetHeaderNumber,
+		TargetHeaderHash,
+		SourceNoncesRange,
+		Proof,
+	>
+	BasicStrategy<
+		SourceHeaderNumber,
+		SourceHeaderHash,
+		TargetHeaderNumber,
+		TargetHeaderHash,
+		SourceNoncesRange,
+		Proof,
+	>
+where
+	SourceHeaderHash: Clone,
+	SourceHeaderNumber: Clone + Ord,
+	SourceNoncesRange: NoncesRange,
+{
+	/// Create new delivery strategy.
+	pub fn new() -> Self {
+		BasicStrategy {
+			source_queue: VecDeque::new(),
+			best_target_nonce: None,
+			_phantom: Default::default(),
+		}
+	}
+
+	/// Reference to source queue.
+	pub(crate) fn source_queue(
+		&self,
+	) -> &VecDeque<(HeaderId<SourceHeaderHash, SourceHeaderNumber>, SourceNoncesRange)> {
+		&self.source_queue
+	}
+
+	/// Mutable reference to source queue to use in tests.
+	#[cfg(test)]
+	pub(crate) fn source_queue_mut(
+		&mut self,
+	) -> &mut VecDeque<(HeaderId<SourceHeaderHash, SourceHeaderNumber>, SourceNoncesRange)> {
+		&mut self.source_queue
+	}
+
+	/// Returns indices of source queue entries, which may be delivered to the target node.
+	///
+	/// The function may skip some nonces from the queue front if nonces from this entry are
+	/// already available at the **best** target block. After this block is finalized, the entry
+	/// will be removed from the queue.
+	///
+	/// All entries before and including the range end index, are guaranteed to be witnessed
+	/// at source blocks that are known to be finalized at the target node.
+	///
+	/// Returns `None` if no entries may be delivered.
+	pub fn available_source_queue_indices<
+		RS: RaceState<
+			HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+			HeaderId<TargetHeaderHash, TargetHeaderNumber>,
+		>,
+	>(
+		&self,
+		race_state: RS,
+	) -> Option<RangeInclusive<usize>> {
+		// if we do not know best nonce at target node, we can't select anything
+		let best_target_nonce = self.best_target_nonce?;
+
+		// if we have already selected nonces that we want to submit, do nothing
+		if race_state.nonces_to_submit().is_some() {
+			return None;
+		}
+
+		// if we already submitted some nonces, do nothing
+		if race_state.nonces_submitted().is_some() {
+			return None;
+		}
+
+		// find first entry that may be delivered to the target node
+		let begin_index = self
+			.source_queue
+			.iter()
+			.enumerate()
+			.skip_while(|(_, (_, nonces))| nonces.end() <= best_target_nonce)
+			.map(|(index, _)| index)
+			.next()?;
+
+		// 1) we want to deliver all nonces, starting from `target_nonce + 1`
+		// 2) we can't deliver new nonce until header, that has emitted this nonce, is finalized
+		// by target client
+		// 3) selector is used for more complicated logic
+		//
+		// => let's first select range of entries inside deque that are already finalized at
+		// the target client and pass this range to the selector
+		let best_header_at_target = race_state.best_finalized_source_header_id_at_best_target()?;
+		let end_index = self
+			.source_queue
+			.iter()
+			.enumerate()
+			.skip(begin_index)
+			.take_while(|(_, (queued_at, _))| queued_at.0 <= best_header_at_target.0)
+			.map(|(index, _)| index)
+			.last()?;
+
+		Some(begin_index..=end_index)
+	}
+
+	/// Remove all nonces that are less than or equal to given nonce from the source queue.
+	fn remove_le_nonces_from_source_queue(&mut self, nonce: MessageNonce) {
+		while let Some((queued_at, queued_range)) = self.source_queue.pop_front() {
+			if let Some(range_to_requeue) = queued_range.greater_than(nonce) {
+				self.source_queue.push_front((queued_at, range_to_requeue));
+				break;
+			}
+		}
+	}
+}
+
+#[async_trait]
+impl<
+		SourceHeaderNumber,
+		SourceHeaderHash,
+		TargetHeaderNumber,
+		TargetHeaderHash,
+		SourceNoncesRange,
+		Proof,
+	>
+	RaceStrategy<
+		HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+		HeaderId<TargetHeaderHash, TargetHeaderNumber>,
+		Proof,
+	>
+	for BasicStrategy<
+		SourceHeaderNumber,
+		SourceHeaderHash,
+		TargetHeaderNumber,
+		TargetHeaderHash,
+		SourceNoncesRange,
+		Proof,
+	>
+where
+	SourceHeaderHash: Clone + Debug + Send + Sync,
+	SourceHeaderNumber: Clone + Ord + Debug + Send + Sync,
+	SourceNoncesRange: NoncesRange + Debug + Send + Sync,
+	TargetHeaderHash: Debug + Send + Sync,
+	TargetHeaderNumber: Debug + Send + Sync,
+	Proof: Debug + Send + Sync,
+{
+	type SourceNoncesRange = SourceNoncesRange;
+	type ProofParameters = ();
+	type TargetNoncesData = ();
+
+	async fn required_source_header_at_target<
+		RS: RaceState<
+			HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+			HeaderId<TargetHeaderHash, TargetHeaderNumber>,
+		>,
+	>(
+		&self,
+		race_state: RS,
+	) -> Option<HeaderId<SourceHeaderHash, SourceHeaderNumber>> {
+		let current_best = race_state.best_finalized_source_header_id_at_best_target()?;
+		self.source_queue
+			.back()
+			.and_then(|(h, _)| if h.0 > current_best.0 { Some(h.clone()) } else { None })
+	}
+
+	fn best_at_source(&self) -> Option<MessageNonce> {
+		let best_in_queue = self.source_queue.back().map(|(_, range)| range.end());
+		match (best_in_queue, self.best_target_nonce) {
+			(Some(best_in_queue), Some(best_target_nonce)) if best_in_queue > best_target_nonce =>
+				Some(best_in_queue),
+			(_, Some(best_target_nonce)) => Some(best_target_nonce),
+			(_, None) => None,
+		}
+	}
+
+	fn best_at_target(&self) -> Option<MessageNonce> {
+		self.best_target_nonce
+	}
+
+	fn source_nonces_updated(
+		&mut self,
+		at_block: HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+		nonces: SourceClientNonces<SourceNoncesRange>,
+	) {
+		let best_in_queue = self
+			.source_queue
+			.back()
+			.map(|(_, range)| range.end())
+			.or(self.best_target_nonce)
+			.unwrap_or_default();
+		self.source_queue.extend(
+			nonces
+				.new_nonces
+				.greater_than(best_in_queue)
+				.into_iter()
+				.map(move |range| (at_block.clone(), range)),
+		)
+	}
+
+	fn reset_best_target_nonces(&mut self) {
+		self.best_target_nonce = None;
+	}
+
+	fn best_target_nonces_updated<
+		RS: RaceState<
+			HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+			HeaderId<TargetHeaderHash, TargetHeaderNumber>,
+		>,
+	>(
+		&mut self,
+		nonces: TargetClientNonces<()>,
+		race_state: &mut RS,
+	) {
+		let nonce = nonces.latest_nonce;
+
+		// if **some** of nonces that we have selected to submit already present at the
+		// target chain => select new nonces
+		let need_to_select_new_nonces = race_state
+			.nonces_to_submit()
+			.map(|nonces| nonce >= *nonces.start())
+			.unwrap_or(false);
+		if need_to_select_new_nonces {
+			tracing::trace!(
+				target: "bridge",
+				%nonce,
+				nonces_to_submit=?race_state.nonces_to_submit(),
+				"Latest nonce at target. Clearing nonces to submit"
+			);
+
+			race_state.reset_nonces_to_submit();
+		}
+
+		// if **some** of nonces that we have submitted already present at the
+		// target chain => select new nonces
+		let need_new_nonces_to_submit = race_state
+			.nonces_submitted()
+			.map(|nonces| nonce >= *nonces.start())
+			.unwrap_or(false);
+		if need_new_nonces_to_submit {
+			tracing::trace!(
+				target: "bridge",
+				%nonce,
+				nonces_submitted=?race_state.nonces_submitted(),
+				"Latest nonce at target. Clearing submitted nonces"
+			);
+
+			race_state.reset_nonces_submitted();
+		}
+
+		self.best_target_nonce = Some(nonce);
+	}
+
+	fn finalized_target_nonces_updated<
+		RS: RaceState<
+			HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+			HeaderId<TargetHeaderHash, TargetHeaderNumber>,
+		>,
+	>(
+		&mut self,
+		nonces: TargetClientNonces<()>,
+		_race_state: &mut RS,
+	) {
+		self.remove_le_nonces_from_source_queue(nonces.latest_nonce);
+		self.best_target_nonce = Some(std::cmp::max(
+			self.best_target_nonce.unwrap_or(nonces.latest_nonce),
+			nonces.latest_nonce,
+		));
+	}
+
+	async fn select_nonces_to_deliver<
+		RS: RaceState<
+			HeaderId<SourceHeaderHash, SourceHeaderNumber>,
+			HeaderId<TargetHeaderHash, TargetHeaderNumber>,
+		>,
+	>(
+		&self,
+		race_state: RS,
+	) -> Option<(RangeInclusive<MessageNonce>, Self::ProofParameters)> {
+		let available_indices = self.available_source_queue_indices(race_state)?;
+		let range_begin = std::cmp::max(
+			self.best_target_nonce? + 1,
+			self.source_queue[*available_indices.start()].1.begin(),
+		);
+		let range_end = self.source_queue[*available_indices.end()].1.end();
+		Some((range_begin..=range_end, ()))
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::{
+		message_lane::{MessageLane, SourceHeaderIdOf, TargetHeaderIdOf},
+		message_lane_loop::tests::{
+			header_id, TestMessageLane, TestMessagesProof, TestSourceHeaderHash,
+			TestSourceHeaderNumber,
+		},
+		message_race_loop::RaceStateImpl,
+	};
+
+	type SourceNoncesRange = RangeInclusive<MessageNonce>;
+
+	type TestRaceStateImpl = RaceStateImpl<
+		SourceHeaderIdOf<TestMessageLane>,
+		TargetHeaderIdOf<TestMessageLane>,
+		TestMessagesProof,
+		(),
+	>;
+
+	type BasicStrategy<P> = super::BasicStrategy<
+		<P as MessageLane>::SourceHeaderNumber,
+		<P as MessageLane>::SourceHeaderHash,
+		<P as MessageLane>::TargetHeaderNumber,
+		<P as MessageLane>::TargetHeaderHash,
+		SourceNoncesRange,
+		<P as MessageLane>::MessagesProof,
+	>;
+
+	fn source_nonces(new_nonces: SourceNoncesRange) -> SourceClientNonces<SourceNoncesRange> {
+		SourceClientNonces { new_nonces, confirmed_nonce: None }
+	}
+
+	fn target_nonces(latest_nonce: MessageNonce) -> TargetClientNonces<()> {
+		TargetClientNonces { latest_nonce, nonces_data: () }
+	}
+
+	#[test]
+	fn best_at_source_is_never_lower_than_target_nonce() {
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		assert_eq!(strategy.best_at_source(), None);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=5));
+		assert_eq!(strategy.best_at_source(), None);
+		strategy.best_target_nonces_updated(target_nonces(10), &mut TestRaceStateImpl::default());
+		assert_eq!(strategy.source_queue, vec![(header_id(1), 1..=5)]);
+		assert_eq!(strategy.best_at_source(), Some(10));
+	}
+
+	#[test]
+	fn source_nonce_is_never_lower_than_known_target_nonce() {
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.best_target_nonces_updated(target_nonces(10), &mut TestRaceStateImpl::default());
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=5));
+		assert_eq!(strategy.source_queue, vec![]);
+	}
+
+	#[test]
+	fn source_nonce_is_never_lower_than_latest_known_source_nonce() {
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=5));
+		strategy.source_nonces_updated(header_id(2), source_nonces(1..=3));
+		strategy.source_nonces_updated(header_id(2), source_nonces(1..=5));
+		assert_eq!(strategy.source_queue, vec![(header_id(1), 1..=5)]);
+	}
+
+	#[test]
+	fn updated_target_nonce_removes_queued_entries() {
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=5));
+		strategy.source_nonces_updated(header_id(2), source_nonces(6..=10));
+		strategy.source_nonces_updated(header_id(3), source_nonces(11..=15));
+		strategy.source_nonces_updated(header_id(4), source_nonces(16..=20));
+		strategy
+			.finalized_target_nonces_updated(target_nonces(15), &mut TestRaceStateImpl::default());
+		assert_eq!(strategy.source_queue, vec![(header_id(4), 16..=20)]);
+		strategy
+			.finalized_target_nonces_updated(target_nonces(17), &mut TestRaceStateImpl::default());
+		assert_eq!(strategy.source_queue, vec![(header_id(4), 18..=20)]);
+	}
+
+	#[test]
+	fn selected_nonces_are_dropped_on_target_nonce_update() {
+		let mut state = TestRaceStateImpl::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		state.nonces_to_submit = Some((header_id(1), 5..=10, (5..=10, None)));
+		// we are going to submit 5..=10, so having latest nonce 4 at target is fine
+		strategy.best_target_nonces_updated(target_nonces(4), &mut state);
+		assert!(state.nonces_to_submit.is_some());
+		// any nonce larger than 4 invalidates the `nonces_to_submit`
+		for nonce in 5..=11 {
+			state.nonces_to_submit = Some((header_id(1), 5..=10, (5..=10, None)));
+			strategy.best_target_nonces_updated(target_nonces(nonce), &mut state);
+			assert!(state.nonces_to_submit.is_none());
+		}
+	}
+
+	#[test]
+	fn submitted_nonces_are_dropped_on_target_nonce_update() {
+		let mut state = TestRaceStateImpl::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		state.nonces_submitted = Some(5..=10);
+		// we have submitted 5..=10, so having latest nonce 4 at target is fine
+		strategy.best_target_nonces_updated(target_nonces(4), &mut state);
+		assert!(state.nonces_submitted.is_some());
+		// any nonce larger than 4 invalidates the `nonces_submitted`
+		for nonce in 5..=11 {
+			state.nonces_submitted = Some(5..=10);
+			strategy.best_target_nonces_updated(target_nonces(nonce), &mut state);
+			assert!(state.nonces_submitted.is_none());
+		}
+	}
+
+	#[async_std::test]
+	async fn nothing_is_selected_if_something_is_already_selected() {
+		let mut state = TestRaceStateImpl::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		state.nonces_to_submit = Some((header_id(1), 1..=10, (1..=10, None)));
+		strategy.best_target_nonces_updated(target_nonces(0), &mut state);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=10));
+		assert_eq!(strategy.select_nonces_to_deliver(state.clone()).await, None);
+	}
+
+	#[async_std::test]
+	async fn nothing_is_selected_if_something_is_already_submitted() {
+		let mut state = TestRaceStateImpl::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		state.nonces_submitted = Some(1..=10);
+		strategy.best_target_nonces_updated(target_nonces(0), &mut state);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=10));
+		assert_eq!(strategy.select_nonces_to_deliver(state.clone()).await, None);
+	}
+
+	#[async_std::test]
+	async fn select_nonces_to_deliver_works() {
+		let mut state = TestRaceStateImpl::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.best_target_nonces_updated(target_nonces(0), &mut state);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=1));
+		strategy.source_nonces_updated(header_id(2), source_nonces(2..=2));
+		strategy.source_nonces_updated(header_id(3), source_nonces(3..=6));
+		strategy.source_nonces_updated(header_id(5), source_nonces(7..=8));
+
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(4));
+		assert_eq!(strategy.select_nonces_to_deliver(state.clone()).await, Some((1..=6, ())));
+		strategy.best_target_nonces_updated(target_nonces(6), &mut state);
+		assert_eq!(strategy.select_nonces_to_deliver(state.clone()).await, None);
+
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(5));
+		assert_eq!(strategy.select_nonces_to_deliver(state.clone()).await, Some((7..=8, ())));
+		strategy.best_target_nonces_updated(target_nonces(8), &mut state);
+		assert_eq!(strategy.select_nonces_to_deliver(state.clone()).await, None);
+	}
+
+	#[test]
+	fn available_source_queue_indices_works() {
+		let mut state = TestRaceStateImpl::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.best_target_nonces_updated(target_nonces(0), &mut state);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=3));
+		strategy.source_nonces_updated(header_id(2), source_nonces(4..=6));
+		strategy.source_nonces_updated(header_id(3), source_nonces(7..=9));
+
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(0));
+		assert_eq!(strategy.available_source_queue_indices(state.clone()), None);
+
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(1));
+		assert_eq!(strategy.available_source_queue_indices(state.clone()), Some(0..=0));
+
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(2));
+		assert_eq!(strategy.available_source_queue_indices(state.clone()), Some(0..=1));
+
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(3));
+		assert_eq!(strategy.available_source_queue_indices(state.clone()), Some(0..=2));
+
+		state.best_finalized_source_header_id_at_best_target = Some(header_id(4));
+		assert_eq!(strategy.available_source_queue_indices(state), Some(0..=2));
+	}
+
+	#[test]
+	fn remove_le_nonces_from_source_queue_works() {
+		let mut state = TestRaceStateImpl::default();
+		let mut strategy = BasicStrategy::<TestMessageLane>::new();
+		strategy.best_target_nonces_updated(target_nonces(0), &mut state);
+		strategy.source_nonces_updated(header_id(1), source_nonces(1..=3));
+		strategy.source_nonces_updated(header_id(2), source_nonces(4..=6));
+		strategy.source_nonces_updated(header_id(3), source_nonces(7..=9));
+
+		fn source_queue_nonces(
+			source_queue: &SourceRangesQueue<
+				TestSourceHeaderHash,
+				TestSourceHeaderNumber,
+				SourceNoncesRange,
+			>,
+		) -> Vec<MessageNonce> {
+			source_queue.iter().flat_map(|(_, range)| range.clone()).collect()
+		}
+
+		strategy.remove_le_nonces_from_source_queue(1);
+		assert_eq!(source_queue_nonces(&strategy.source_queue), vec![2, 3, 4, 5, 6, 7, 8, 9],);
+
+		strategy.remove_le_nonces_from_source_queue(5);
+		assert_eq!(source_queue_nonces(&strategy.source_queue), vec![6, 7, 8, 9],);
+
+		strategy.remove_le_nonces_from_source_queue(9);
+		assert_eq!(source_queue_nonces(&strategy.source_queue), Vec::<MessageNonce>::new(),);
+
+		strategy.remove_le_nonces_from_source_queue(100);
+		assert_eq!(source_queue_nonces(&strategy.source_queue), Vec::<MessageNonce>::new(),);
+	}
+
+	#[async_std::test]
+	async fn previous_nonces_are_selected_if_reorg_happens_at_target_chain() {
+		let source_header_1 = header_id(1);
+		let target_header_1 = header_id(1);
+
+		// we start in perfect sync state - all headers are synced and finalized on both ends
+		let mut state = TestRaceStateImpl {
+			best_finalized_source_header_id_at_source: Some(source_header_1),
+			best_finalized_source_header_id_at_best_target: Some(source_header_1),
+			best_target_header_id: Some(target_header_1),
+			best_finalized_target_header_id: Some(target_header_1),
+			nonces_to_submit: None,
+			nonces_to_submit_batch: None,
+			nonces_submitted: None,
+		};
+
+		// in this state we have 1 available nonce for delivery
+		let mut strategy = BasicStrategy::<TestMessageLane> {
+			source_queue: vec![(header_id(1), 1..=1)].into_iter().collect(),
+			best_target_nonce: Some(0),
+			_phantom: PhantomData,
+		};
+		assert_eq!(strategy.select_nonces_to_deliver(state.clone()).await, Some((1..=1, ())),);
+
+		// let's say we have submitted 1..=1
+		state.nonces_submitted = Some(1..=1);
+
+		// then new nonce 2 appear at the source block 2
+		let source_header_2 = header_id(2);
+		state.best_finalized_source_header_id_at_source = Some(source_header_2);
+		strategy.source_nonces_updated(
+			source_header_2,
+			SourceClientNonces { new_nonces: 2..=2, confirmed_nonce: None },
+		);
+		// and nonce 1 appear at the best block of the target node (best finalized still has 0
+		// nonces)
+		let target_header_2 = header_id(2);
+		state.best_target_header_id = Some(target_header_2);
+		strategy.best_target_nonces_updated(
+			TargetClientNonces { latest_nonce: 1, nonces_data: () },
+			&mut state,
+		);
+
+		// then best target header is retracted
+		strategy.best_target_nonces_updated(
+			TargetClientNonces { latest_nonce: 0, nonces_data: () },
+			&mut state,
+		);
+
+		// ... and some fork with zero delivered nonces is finalized
+		let target_header_2_fork = header_id(2_1);
+		state.best_finalized_source_header_id_at_source = Some(source_header_2);
+		state.best_finalized_source_header_id_at_best_target = Some(source_header_2);
+		state.best_target_header_id = Some(target_header_2_fork);
+		state.best_finalized_target_header_id = Some(target_header_2_fork);
+		strategy.finalized_target_nonces_updated(
+			TargetClientNonces { latest_nonce: 0, nonces_data: () },
+			&mut state,
+		);
+
+		// now we have to select nonce 1 for delivery again
+		assert_eq!(strategy.select_nonces_to_deliver(state.clone()).await, Some((1..=2, ())),);
+	}
+}

pezbridges/relays/messages/src/metrics.rs

@@ -0,0 +1,177 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Metrics for message lane relay loop.
+
+use crate::{
+	message_lane::MessageLane,
+	message_lane_loop::{SourceClientState, TargetClientState},
+};
+
+use bp_messages::{HashedLaneId, LegacyLaneId, MessageNonce};
+use pez_finality_relay::SyncLoopMetrics;
+use relay_utils::metrics::{
+	metric_name, register, GaugeVec, Metric, Opts, PrometheusError, Registry, U64,
+};
+
+/// Message lane relay metrics.
+///
+/// Cloning only clones references.
+#[derive(Clone)]
+pub struct MessageLaneLoopMetrics {
+	/// Best finalized block numbers - "source", "source_at_target", "target_at_source".
+	source_to_target_finality_metrics: SyncLoopMetrics,
+	/// Best finalized block numbers - "source", "target", "source_at_target", "target_at_source".
+	target_to_source_finality_metrics: SyncLoopMetrics,
+	/// Lane state nonces: "source_latest_generated", "source_latest_confirmed",
+	/// "target_latest_received", "target_latest_confirmed".
+	lane_state_nonces: GaugeVec<U64>,
+}
+
+impl MessageLaneLoopMetrics {
+	/// Create and register messages loop metrics.
+	pub fn new(prefix: Option<&str>) -> Result<Self, PrometheusError> {
+		Ok(MessageLaneLoopMetrics {
+			source_to_target_finality_metrics: SyncLoopMetrics::new(
+				prefix,
+				"source",
+				"source_at_target",
+			)?,
+			target_to_source_finality_metrics: SyncLoopMetrics::new(
+				prefix,
+				"target",
+				"target_at_source",
+			)?,
+			lane_state_nonces: GaugeVec::new(
+				Opts::new(metric_name(prefix, "lane_state_nonces"), "Nonces of the lane state"),
+				&["type"],
+			)?,
+		})
+	}
+
+	/// Update source client state metrics.
+	pub fn update_source_state<P: MessageLane>(&self, source_client_state: SourceClientState<P>) {
+		self.source_to_target_finality_metrics
+			.update_best_block_at_source(source_client_state.best_self.0);
+		if let Some(best_finalized_peer_at_best_self) =
+			source_client_state.best_finalized_peer_at_best_self
+		{
+			self.target_to_source_finality_metrics
+				.update_best_block_at_target(best_finalized_peer_at_best_self.0);
+			if let Some(actual_best_finalized_peer_at_best_self) =
+				source_client_state.actual_best_finalized_peer_at_best_self
+			{
+				self.target_to_source_finality_metrics.update_using_same_fork(
+					best_finalized_peer_at_best_self.1 == actual_best_finalized_peer_at_best_self.1,
+				);
+			}
+		}
+	}
+
+	/// Update target client state metrics.
+	pub fn update_target_state<P: MessageLane>(&self, target_client_state: TargetClientState<P>) {
+		self.target_to_source_finality_metrics
+			.update_best_block_at_source(target_client_state.best_self.0);
+		if let Some(best_finalized_peer_at_best_self) =
+			target_client_state.best_finalized_peer_at_best_self
+		{
+			self.source_to_target_finality_metrics
+				.update_best_block_at_target(best_finalized_peer_at_best_self.0);
+			if let Some(actual_best_finalized_peer_at_best_self) =
+				target_client_state.actual_best_finalized_peer_at_best_self
+			{
+				self.source_to_target_finality_metrics.update_using_same_fork(
+					best_finalized_peer_at_best_self.1 == actual_best_finalized_peer_at_best_self.1,
+				);
+			}
+		}
+	}
+
+	/// Update latest generated nonce at source.
+	pub fn update_source_latest_generated_nonce(
+		&self,
+		source_latest_generated_nonce: MessageNonce,
+	) {
+		self.lane_state_nonces
+			.with_label_values(&["source_latest_generated"])
+			.set(source_latest_generated_nonce);
+	}
+
+	/// Update the latest confirmed nonce at source.
+	pub fn update_source_latest_confirmed_nonce(
+		&self,
+		source_latest_confirmed_nonce: MessageNonce,
+	) {
+		self.lane_state_nonces
+			.with_label_values(&["source_latest_confirmed"])
+			.set(source_latest_confirmed_nonce);
+	}
+
+	/// Update the latest received nonce at target.
+	pub fn update_target_latest_received_nonce(&self, target_latest_generated_nonce: MessageNonce) {
+		self.lane_state_nonces
+			.with_label_values(&["target_latest_received"])
+			.set(target_latest_generated_nonce);
+	}
+
+	/// Update the latest confirmed nonce at target.
+	pub fn update_target_latest_confirmed_nonce(
+		&self,
+		target_latest_confirmed_nonce: MessageNonce,
+	) {
+		self.lane_state_nonces
+			.with_label_values(&["target_latest_confirmed"])
+			.set(target_latest_confirmed_nonce);
+	}
+}
+
+impl Metric for MessageLaneLoopMetrics {
+	fn register(&self, registry: &Registry) -> Result<(), PrometheusError> {
+		self.source_to_target_finality_metrics.register(registry)?;
+		self.target_to_source_finality_metrics.register(registry)?;
+		register(self.lane_state_nonces.clone(), registry)?;
+		Ok(())
+	}
+}
+
+/// Provides a label for metrics.
+pub trait Labeled {
+	/// Returns a label.
+	fn label(&self) -> String;
+}
+
+/// `Labeled` implementation for `LegacyLaneId`.
+impl Labeled for LegacyLaneId {
+	fn label(&self) -> String {
+		hex::encode(self.0)
+	}
+}
+
+/// `Labeled` implementation for `HashedLaneId`.
+impl Labeled for HashedLaneId {
+	fn label(&self) -> String {
+		format!("{:?}", self.inner())
+	}
+}
+
+#[test]
+fn lane_to_label_works() {
+	assert_eq!(
+		"0x0101010101010101010101010101010101010101010101010101010101010101",
+		HashedLaneId::from_inner(pezsp_core::H256::from([1u8; 32])).label(),
+	);
+	assert_eq!("00000001", LegacyLaneId([0, 0, 0, 1]).label());
+}

pezbridges/relays/teyrchains/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "teyrchains-relay"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+repository.workspace = true
+publish = false
+description = "Pezkuwi SDK component: teyrchains relay"
+documentation = "https://docs.rs/teyrchains-relay"
+homepage = { workspace = true }
+
+[lints]
+workspace = true
+
+[dependencies]
+async-std = { workspace = true }
+async-trait = { workspace = true }
+futures = { workspace = true }
+relay-utils = { workspace = true }
+tracing = { workspace = true }
+
+# Bridge dependencies
+bp-pezkuwi-core = { workspace = true, default-features = true }
+relay-bizinikiwi-client = { workspace = true }
+
+[dev-dependencies]
+relay-bizinikiwi-client = { features = ["test-helpers"], workspace = true }
+pezsp-core = { workspace = true, default-features = true }
+
+[features]
+runtime-benchmarks = [
+  "bp-pezkuwi-core/runtime-benchmarks",
+  "relay-bizinikiwi-client/runtime-benchmarks",
+  "relay-utils/runtime-benchmarks",
+]

pezbridges/relays/teyrchains/README.md

@@ -0,0 +1,50 @@
+# Teyrchains Finality Relay
+
+The teyrchains finality relay works with two chains - source relay chain and target chain (which may be standalone
+chain, relay chain or a teyrchain). The source chain must have the
+[`paras` pezpallet](https://github.com/paritytech/polkadot/tree/master/runtime/parachains/src/paras) deployed at its
+runtime. The target chain must have the [bridge teyrchains pezpallet](../../modules/teyrchains/) deployed at its runtime.
+
+The relay is configured to submit heads of one or several teyrchains. It pokes source chain periodically and compares
+teyrchain heads that are known to the source relay chain to heads at the target chain. If there are new heads,
+the relay submits them to the target chain.
+
+More: [Teyrchains Finality Relay Sequence Diagram](../../docs/teyrchains-pez-finality-relay.html).
+
+## How to Use the Teyrchains Finality Relay
+
+There are only two traits that need to be implemented. The [`SourceChain`](./src/teyrchains_loop.rs) implementation
+is supposed to connect to the source chain node. It must be able to read teyrchain heads from the `Heads` map of
+the [`paras` pezpallet](https://github.com/paritytech/polkadot/tree/master/runtime/parachains/src/paras).
+It also must create storage proofs of `Heads` map entries, when required.
+
+The [`TargetChain`](./src/teyrchains_loop.rs) implementation connects to the target chain node. It must be able
+to return the best known head of given teyrchain. When required, it must be able to craft and submit teyrchains
+finality delivery transaction to the target node.
+
+The main entrypoint for the crate is the [`run` function](./src/teyrchains_loop.rs), which takes source and target
+clients and [`TeyrchainSyncParams`](./src/teyrchains_loop.rs) parameters. The most important parameter is the
+`teyrchains` - it is the set of teyrchains, which relay tracks and updates. The other important parameter that
+may affect the relay operational costs is the `strategy`. If it is set to `Any`, then the finality delivery
+transaction is submitted if at least one of tracked teyrchain heads is updated. The other option is `All`. Then
+the relay waits until all tracked teyrchain heads are updated and submits them all in a single finality delivery
+transaction.
+
+## Teyrchain Finality Relay Metrics
+
+Every teyrchain in PezkuwiChain is identified by the 32-bit number. All metrics, exposed by the teyrchains finality
+relay have the `teyrchain` label, which is set to the teyrchain id. And the metrics are prefixed with the prefix,
+that depends on the name of the source relay and target chains. The list below shows metrics names for
+pezkuwichain (source relay chain) to BridgeHubzagros (target chain) teyrchains finality relay. For other chains, simply
+change chain names. So the metrics are:
+
+- `pezkuwichain_to_BridgeHubzagros_Teyrchains_best_teyrchain_block_number_at_source` - returns best known teyrchain block
+  number, registered in the `paras` pezpallet at the source relay chain (pezkuwichain in our example);
+
+- `pezkuwichain_to_BridgeHubzagros_Teyrchains_best_teyrchain_block_number_at_target` - returns best known teyrchain block
+  number, registered in the bridge teyrchains pezpallet at the target chain (BridgeHubzagros in our example).
+
+If relay operates properly, you should see that
+the `pezkuwichain_to_BridgeHubzagros_Teyrchains_best_teyrchain_block_number_at_target` tries to reach
+the `pezkuwichain_to_BridgeHubzagros_Teyrchains_best_teyrchain_block_number_at_source`.
+And the latter one always increases.

pezbridges/relays/teyrchains/src/lib.rs

@@ -0,0 +1,32 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::fmt::Debug;
+
+use relay_bizinikiwi_client::{Chain, Teyrchain};
+
+pub mod teyrchains_loop;
+pub mod teyrchains_loop_metrics;
+
+/// Finality proofs synchronization pipeline.
+pub trait TeyrchainsPipeline: 'static + Clone + Debug + Send + Sync {
+	/// Relay chain which is storing teyrchain heads in its `paras` module.
+	type SourceRelayChain: Chain;
+	/// Teyrchain which headers we are syncing here.
+	type SourceTeyrchain: Teyrchain;
+	/// Target chain (either relay or para) which wants to know about new teyrchain heads.
+	type TargetChain: Chain;
+}

pezbridges/relays/teyrchains/src/teyrchains_loop_metrics.rs

@@ -0,0 +1,86 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use bp_pezkuwi_core::teyrchains::ParaId;
+use relay_utils::{
+	metrics::{metric_name, register, Gauge, Metric, PrometheusError, Registry, U64},
+	UniqueSaturatedInto,
+};
+
+/// Teyrchains sync metrics.
+#[derive(Clone)]
+pub struct TeyrchainsLoopMetrics {
+	/// Best teyrchains header numbers at the source.
+	best_source_block_numbers: Gauge<U64>,
+	/// Best teyrchains header numbers at the target.
+	best_target_block_numbers: Gauge<U64>,
+}
+
+impl TeyrchainsLoopMetrics {
+	/// Create and register teyrchains loop metrics.
+	pub fn new(prefix: Option<&str>) -> Result<Self, PrometheusError> {
+		Ok(TeyrchainsLoopMetrics {
+			best_source_block_numbers: Gauge::new(
+				metric_name(prefix, "best_teyrchain_block_number_at_source"),
+				"Best teyrchain block numbers at the source relay chain".to_string(),
+			)?,
+			best_target_block_numbers: Gauge::new(
+				metric_name(prefix, "best_teyrchain_block_number_at_target"),
+				"Best teyrchain block numbers at the target chain".to_string(),
+			)?,
+		})
+	}
+
+	/// Update best block number at source.
+	pub fn update_best_teyrchain_block_at_source<Number: UniqueSaturatedInto<u64>>(
+		&self,
+		teyrchain: ParaId,
+		block_number: Number,
+	) {
+		let block_number = block_number.unique_saturated_into();
+		tracing::trace!(
+			target: "bridge-metrics",
+			?teyrchain,
+			?block_number,
+			"Updated value of metric 'best_teyrchain_block_number_at_source"
+		);
+		self.best_source_block_numbers.set(block_number);
+	}
+
+	/// Update best block number at target.
+	pub fn update_best_teyrchain_block_at_target<Number: UniqueSaturatedInto<u64>>(
+		&self,
+		teyrchain: ParaId,
+		block_number: Number,
+	) {
+		let block_number = block_number.unique_saturated_into();
+		tracing::trace!(
+			target: "bridge-metrics",
+			?teyrchain,
+			?block_number,
+			"Updated value of metric 'best_teyrchain_block_number_at_target"
+		);
+		self.best_target_block_numbers.set(block_number);
+	}
+}
+
+impl Metric for TeyrchainsLoopMetrics {
+	fn register(&self, registry: &Registry) -> Result<(), PrometheusError> {
+		register(self.best_source_block_numbers.clone(), registry)?;
+		register(self.best_target_block_numbers.clone(), registry)?;
+		Ok(())
+	}
+}

pezbridges/relays/utils/Cargo.toml

@@ -0,0 +1,45 @@
+[package]
+name = "relay-utils"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+repository.workspace = true
+publish = false
+description = "Pezkuwi SDK component: relay utils"
+documentation = "https://docs.rs/relay-utils"
+homepage = { workspace = true }
+
+[lints]
+workspace = true
+
+[dependencies]
+anyhow = { workspace = true, default-features = true }
+async-std = { workspace = true }
+async-trait = { workspace = true }
+backoff = { workspace = true }
+futures = { workspace = true }
+isahc = { workspace = true }
+jsonpath_lib = { workspace = true }
+num-traits = { workspace = true, default-features = true }
+parking_lot = { workspace = true, default-features = true }
+serde_json = { workspace = true, default-features = true }
+pezsp-tracing = { workspace = true, default-features = true }
+sysinfo = { workspace = true }
+thiserror = { workspace = true }
+time = { features = ["formatting", "local-offset", "std"], workspace = true }
+tokio = { features = ["rt"], workspace = true, default-features = true }
+tracing = { workspace = true }
+
+# Bridge dependencies
+pezbp-runtime = { workspace = true, default-features = true }
+
+# Bizinikiwi dependencies
+prometheus-endpoint = { workspace = true, default-features = true }
+pezsp-runtime = { workspace = true, default-features = true }
+
+[features]
+runtime-benchmarks = [
+  "pezbp-runtime/runtime-benchmarks",
+  "pezsp-runtime/runtime-benchmarks",
+]

pezbridges/relays/utils/src/error.rs

@@ -0,0 +1,46 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::net::AddrParseError;
+use thiserror::Error;
+
+/// Result type used by relay utilities.
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Relay utilities errors.
+#[derive(Error, Debug)]
+pub enum Error {
+	/// Failed to request a float value from HTTP service.
+	#[error("Failed to fetch token price from remote server: {0}")]
+	FetchTokenPrice(#[source] anyhow::Error),
+	/// Failed to parse the response from HTTP service.
+	#[error("Failed to parse HTTP service response: {0:?}. Response: {1:?}")]
+	ParseHttp(serde_json::Error, String),
+	/// Failed to select response value from the Json response.
+	#[error("Failed to select value from response: {0:?}. Response: {1:?}")]
+	SelectResponseValue(jsonpath_lib::JsonPathError, String),
+	/// Failed to parse float value from the selected value.
+	#[error(
+		"Failed to parse float value {0:?} from response. It is assumed to be positive and normal"
+	)]
+	ParseFloat(f64),
+	/// Couldn't found value in the JSON response.
+	#[error("Missing required value from response: {0:?}")]
+	MissingResponseValue(String),
+	/// Invalid host address was used for exposing Prometheus metrics.
+	#[error("Invalid host {0} is used to expose Prometheus metrics: {1}")]
+	ExposingMetricsInvalidHost(String, AddrParseError),
+	/// Prometheus error.
+	#[error("{0}")]
+	Prometheus(#[from] prometheus_endpoint::prometheus::Error),
+}

pezbridges/relays/utils/src/initialize.rs

@@ -0,0 +1,72 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Relayer initialization functions.
+
+use parking_lot::Mutex;
+use pezsp_tracing::{
+	tracing::Level,
+	tracing_subscriber::{
+		fmt::{time::OffsetTime, SubscriberBuilder},
+		EnvFilter,
+	},
+};
+use std::cell::RefCell;
+
+/// Relayer version that is provided as metric. Must be set by a binary
+/// (get it with `option_env!("CARGO_PKG_VERSION")` from a binary package code).
+pub static RELAYER_VERSION: Mutex<Option<String>> = Mutex::new(None);
+
+async_std::task_local! {
+	pub(crate) static LOOP_NAME: RefCell<String> = RefCell::new(String::default());
+}
+
+/// Initialize relay environment.
+pub fn initialize_relay() {
+	initialize_logger(true);
+}
+
+/// Initialize Relay logger instance.
+pub fn initialize_logger(with_timestamp: bool) {
+	let format = time::format_description::parse(
+		"[year]-[month]-[day] \
+		[hour repr:24]:[minute]:[second] [offset_hour sign:mandatory]",
+	)
+	.expect("static format string is valid");
+
+	let local_time = OffsetTime::new(
+		time::UtcOffset::current_local_offset().unwrap_or(time::UtcOffset::UTC),
+		format,
+	);
+
+	let env_filter = EnvFilter::builder()
+		.with_default_directive(Level::WARN.into())
+		.with_default_directive("bridge=info".parse().expect("static filter string is valid"))
+		.from_env_lossy();
+
+	let builder = SubscriberBuilder::default().with_env_filter(env_filter);
+
+	if with_timestamp {
+		builder.with_timer(local_time).init();
+	} else {
+		builder.without_time().init();
+	}
+}
+
+/// Initialize relay loop. Must only be called once per every loop task.
+pub(crate) fn initialize_loop(loop_name: String) {
+	LOOP_NAME.with(|g_loop_name| *g_loop_name.borrow_mut() = loop_name);
+}

pezbridges/relays/utils/src/lib.rs

@@ -0,0 +1,318 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Utilities used by different relays.
+
+pub use pezbp_runtime::HeaderId;
+pub use error::Error;
+pub use relay_loop::{relay_loop, relay_metrics};
+pub use pezsp_runtime::traits::{UniqueSaturatedFrom, UniqueSaturatedInto};
+use std::fmt::Debug;
+
+use async_trait::async_trait;
+use backoff::{backoff::Backoff, ExponentialBackoff};
+use futures::future::{BoxFuture, FutureExt};
+use std::time::Duration;
+use thiserror::Error;
+
+/// Default relay loop stall timeout. If transactions generated by relay are immortal, then
+/// this timeout is used.
+///
+/// There are no any strict requirements on block time in Bizinikiwi. But we assume here that all
+/// Bizinikiwi-based chains will be designed to produce relatively fast (compared to the slowest
+/// blockchains) blocks. So 1 hour seems to be a good guess for (even congested) chains to mine
+/// transaction, or remove it from the pool.
+pub const STALL_TIMEOUT: Duration = Duration::from_secs(60 * 60);
+
+/// Max delay after connection-unrelated error happened before we'll try the
+/// same request again.
+pub const MAX_BACKOFF_INTERVAL: Duration = Duration::from_secs(60);
+/// Delay after connection-related error happened before we'll try
+/// reconnection again.
+pub const CONNECTION_ERROR_DELAY: Duration = Duration::from_secs(10);
+
+pub mod error;
+pub mod initialize;
+pub mod metrics;
+pub mod relay_loop;
+
+/// Block number traits shared by all chains that relay is able to serve.
+pub trait BlockNumberBase:
+	'static
+	+ From<u32>
+	+ UniqueSaturatedInto<u64>
+	+ Ord
+	+ Clone
+	+ Copy
+	+ Default
+	+ Send
+	+ Sync
+	+ std::fmt::Debug
+	+ std::fmt::Display
+	+ std::hash::Hash
+	+ std::ops::Add<Output = Self>
+	+ std::ops::Sub<Output = Self>
+	+ num_traits::CheckedSub
+	+ num_traits::Saturating
+	+ num_traits::Zero
+	+ num_traits::One
+{
+}
+
+impl<T> BlockNumberBase for T where
+	T: 'static
+		+ From<u32>
+		+ UniqueSaturatedInto<u64>
+		+ Ord
+		+ Clone
+		+ Copy
+		+ Default
+		+ Send
+		+ Sync
+		+ std::fmt::Debug
+		+ std::fmt::Display
+		+ std::hash::Hash
+		+ std::ops::Add<Output = Self>
+		+ std::ops::Sub<Output = Self>
+		+ num_traits::CheckedSub
+		+ num_traits::Saturating
+		+ num_traits::Zero
+		+ num_traits::One
+{
+}
+
+/// Macro that returns (client, Err(error)) tuple from function if result is Err(error).
+#[macro_export]
+macro_rules! bail_on_error {
+	($result: expr) => {
+		match $result {
+			(client, Ok(result)) => (client, result),
+			(client, Err(error)) => return (client, Err(error)),
+		}
+	};
+}
+
+/// Macro that returns (client, Err(error)) tuple from function if result is Err(error).
+#[macro_export]
+macro_rules! bail_on_arg_error {
+	($result: expr, $client: ident) => {
+		match $result {
+			Ok(result) => result,
+			Err(error) => return ($client, Err(error)),
+		}
+	};
+}
+
+/// Error type that can signal connection errors.
+pub trait MaybeConnectionError {
+	/// Returns true if error (maybe) represents connection error.
+	fn is_connection_error(&self) -> bool;
+}
+
+/// Final status of the tracked transaction.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum TrackedTransactionStatus<BlockId> {
+	/// Transaction has been lost.
+	Lost,
+	/// Transaction has been mined and finalized at given block.
+	Finalized(BlockId),
+}
+
+/// Transaction tracker.
+#[async_trait]
+pub trait TransactionTracker: Send {
+	/// Header id, used by the chain.
+	type HeaderId: Clone + Debug + Send;
+
+	/// Wait until transaction is either finalized or invalidated/lost.
+	async fn wait(self) -> TrackedTransactionStatus<Self::HeaderId>;
+}
+
+/// Future associated with `TransactionTracker`, monitoring the transaction status.
+pub type TrackedTransactionFuture<'a, T> =
+	BoxFuture<'a, TrackedTransactionStatus<<T as TransactionTracker>::HeaderId>>;
+
+/// Stringified error that may be either connection-related or not.
+#[derive(Error, Debug)]
+pub enum StringifiedMaybeConnectionError {
+	/// The error is connection-related error.
+	#[error("{0}")]
+	Connection(String),
+	/// The error is connection-unrelated error.
+	#[error("{0}")]
+	NonConnection(String),
+}
+
+impl StringifiedMaybeConnectionError {
+	/// Create new stringified connection error.
+	pub fn new(is_connection_error: bool, error: String) -> Self {
+		if is_connection_error {
+			StringifiedMaybeConnectionError::Connection(error)
+		} else {
+			StringifiedMaybeConnectionError::NonConnection(error)
+		}
+	}
+}
+
+impl MaybeConnectionError for StringifiedMaybeConnectionError {
+	fn is_connection_error(&self) -> bool {
+		match *self {
+			StringifiedMaybeConnectionError::Connection(_) => true,
+			StringifiedMaybeConnectionError::NonConnection(_) => false,
+		}
+	}
+}
+
+/// Exponential backoff for connection-unrelated errors retries.
+pub fn retry_backoff() -> ExponentialBackoff {
+	ExponentialBackoff {
+		// we do not want relayer to stop
+		max_elapsed_time: None,
+		max_interval: MAX_BACKOFF_INTERVAL,
+		..Default::default()
+	}
+}
+
+/// Compact format of IDs vector.
+pub fn format_ids<Id: std::fmt::Debug>(mut ids: impl ExactSizeIterator<Item = Id>) -> String {
+	const NTH_PROOF: &str = "we have checked len; qed";
+	match ids.len() {
+		0 => "<nothing>".into(),
+		1 => format!("{:?}", ids.next().expect(NTH_PROOF)),
+		2 => {
+			let id0 = ids.next().expect(NTH_PROOF);
+			let id1 = ids.next().expect(NTH_PROOF);
+			format!("[{id0:?}, {id1:?}]")
+		},
+		len => {
+			let id0 = ids.next().expect(NTH_PROOF);
+			let id_last = ids.last().expect(NTH_PROOF);
+			format!("{len}:[{id0:?} ... {id_last:?}]")
+		},
+	}
+}
+
+/// Stream that emits item every `timeout_ms` milliseconds.
+pub fn interval(timeout: Duration) -> impl futures::Stream<Item = ()> {
+	futures::stream::unfold((), move |_| async move {
+		async_std::task::sleep(timeout).await;
+		Some(((), ()))
+	})
+}
+
+/// Which client has caused error.
+#[derive(Debug, Eq, Clone, Copy, PartialEq)]
+pub enum FailedClient {
+	/// It is the source client who has caused error.
+	Source,
+	/// It is the target client who has caused error.
+	Target,
+	/// Both clients are failing, or we just encountered some other error that
+	/// should be treated like that.
+	Both,
+}
+
+/// Future process result.
+#[derive(Debug, Clone, Copy)]
+pub enum ProcessFutureResult {
+	/// Future has been processed successfully.
+	Success,
+	/// Future has failed with non-connection error.
+	Failed,
+	/// Future has failed with connection error.
+	ConnectionFailed,
+}
+
+impl ProcessFutureResult {
+	/// Returns true if result is Success.
+	pub fn is_ok(self) -> bool {
+		match self {
+			ProcessFutureResult::Success => true,
+			ProcessFutureResult::Failed | ProcessFutureResult::ConnectionFailed => false,
+		}
+	}
+
+	/// Returns `Ok(())` if future has succeeded.
+	/// Returns `Err(failed_client)` otherwise.
+	pub fn fail_if_error(self, failed_client: FailedClient) -> Result<(), FailedClient> {
+		if self.is_ok() {
+			Ok(())
+		} else {
+			Err(failed_client)
+		}
+	}
+
+	/// Returns Ok(true) if future has succeeded.
+	/// Returns Ok(false) if future has failed with non-connection error.
+	/// Returns Err if future is `ConnectionFailed`.
+	pub fn fail_if_connection_error(
+		self,
+		failed_client: FailedClient,
+	) -> Result<bool, FailedClient> {
+		match self {
+			ProcessFutureResult::Success => Ok(true),
+			ProcessFutureResult::Failed => Ok(false),
+			ProcessFutureResult::ConnectionFailed => Err(failed_client),
+		}
+	}
+}
+
+/// Process result of the future from a client.
+pub fn process_future_result<TResult, TError, TGoOfflineFuture>(
+	result: Result<TResult, TError>,
+	retry_backoff: &mut ExponentialBackoff,
+	on_success: impl FnOnce(TResult),
+	go_offline_future: &mut std::pin::Pin<&mut futures::future::Fuse<TGoOfflineFuture>>,
+	go_offline: impl FnOnce(Duration) -> TGoOfflineFuture,
+	error_pattern: impl FnOnce() -> String,
+) -> ProcessFutureResult
+where
+	TError: std::fmt::Debug + MaybeConnectionError,
+	TGoOfflineFuture: FutureExt,
+{
+	match result {
+		Ok(result) => {
+			on_success(result);
+			retry_backoff.reset();
+			ProcessFutureResult::Success
+		},
+		Err(error) if error.is_connection_error() => {
+			tracing::error!(
+				target: "bridge",
+				?error,
+				error_pattern=?error_pattern(),
+				"Going to restart"
+			);
+
+			retry_backoff.reset();
+			go_offline_future.set(go_offline(CONNECTION_ERROR_DELAY).fuse());
+			ProcessFutureResult::ConnectionFailed
+		},
+		Err(error) => {
+			let retry_delay = retry_backoff.next_backoff().unwrap_or(CONNECTION_ERROR_DELAY);
+			tracing::error!(
+				target: "bridge",
+				?error,
+				error_pattern=?error_pattern(),
+				retry_as_secs=%retry_delay.as_secs_f64(),
+				"Retrying"
+			);
+
+			go_offline_future.set(go_offline(retry_delay).fuse());
+			ProcessFutureResult::Failed
+		},
+	}
+}

pezbridges/relays/utils/src/metrics.rs

@@ -0,0 +1,192 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+pub use float_json_value::FloatJsonValueMetric;
+pub use global::GlobalMetrics;
+pub use prometheus_endpoint::{
+	prometheus::core::{Atomic, Collector},
+	register, Counter, CounterVec, Gauge, GaugeVec, Opts, PrometheusError, Registry, F64, I64, U64,
+};
+
+use async_std::sync::{Arc, RwLock};
+use async_trait::async_trait;
+use std::{fmt::Debug, time::Duration};
+
+mod float_json_value;
+mod global;
+
+/// Shared reference to `f64` value that is updated by the metric.
+pub type F64SharedRef = Arc<RwLock<Option<f64>>>;
+/// Int gauge metric type.
+pub type IntGauge = Gauge<U64>;
+
+/// Unparsed address that needs to be used to expose Prometheus metrics.
+#[derive(Debug, Clone)]
+pub struct MetricsAddress {
+	/// Serve HTTP requests at given host.
+	pub host: String,
+	/// Serve HTTP requests at given port.
+	pub port: u16,
+}
+
+/// Prometheus endpoint MetricsParams.
+#[derive(Debug, Clone)]
+pub struct MetricsParams {
+	/// Interface and TCP port to be used when exposing Prometheus metrics.
+	pub address: Option<MetricsAddress>,
+	/// Metrics registry. May be `Some(_)` if several components share the same endpoint.
+	pub registry: Registry,
+}
+
+/// Metric API.
+pub trait Metric: Clone + Send + Sync + 'static {
+	fn register(&self, registry: &Registry) -> Result<(), PrometheusError>;
+}
+
+/// Standalone metric API.
+///
+/// Metrics of this kind know how to update themselves, so we may just spawn and forget the
+/// asynchronous self-update task.
+#[async_trait]
+pub trait StandaloneMetric: Metric {
+	/// Update metric values.
+	async fn update(&self);
+
+	/// Metrics update interval.
+	fn update_interval(&self) -> Duration;
+
+	/// Register and spawn metric. Metric is only spawned if it is registered for the first time.
+	fn register_and_spawn(self, registry: &Registry) -> Result<(), PrometheusError> {
+		match self.register(registry) {
+			Ok(()) => {
+				self.spawn();
+				Ok(())
+			},
+			Err(PrometheusError::AlreadyReg) => Ok(()),
+			Err(e) => Err(e),
+		}
+	}
+
+	/// Spawn the self update task that will keep update metric value at given intervals.
+	fn spawn(self) {
+		async_std::task::spawn(async move {
+			let update_interval = self.update_interval();
+			loop {
+				self.update().await;
+				async_std::task::sleep(update_interval).await;
+			}
+		});
+	}
+}
+
+impl Default for MetricsAddress {
+	fn default() -> Self {
+		MetricsAddress { host: "127.0.0.1".into(), port: 9616 }
+	}
+}
+
+impl MetricsParams {
+	/// Creates metrics params from metrics address.
+	pub fn new(
+		address: Option<MetricsAddress>,
+		relay_version: String,
+		relay_commit: String,
+	) -> Result<Self, PrometheusError> {
+		const BUILD_INFO_METRIC: &str = "bizinikiwi_relay_build_info";
+
+		let registry = Registry::new();
+		register(
+			Gauge::<U64>::with_opts(
+				Opts::new(
+					BUILD_INFO_METRIC,
+					"A metric with a constant '1' value labeled by version",
+				)
+				.const_label("version", &relay_version)
+				.const_label("commit", &relay_commit),
+			)?,
+			&registry,
+		)?
+		.set(1);
+
+		tracing::info!(
+			target: "bridge",
+			metric=%BUILD_INFO_METRIC,
+			version=%relay_version,
+			commit=%relay_commit,
+			"Exposed metric"
+		);
+
+		Ok(MetricsParams { address, registry })
+	}
+
+	/// Creates metrics params so that metrics are not exposed.
+	pub fn disabled() -> Self {
+		MetricsParams { address: None, registry: Registry::new() }
+	}
+
+	/// Do not expose metrics.
+	#[must_use]
+	pub fn disable(mut self) -> Self {
+		self.address = None;
+		self
+	}
+}
+
+/// Returns metric name optionally prefixed with given prefix.
+pub fn metric_name(prefix: Option<&str>, name: &str) -> String {
+	if let Some(prefix) = prefix {
+		format!("{prefix}_{name}")
+	} else {
+		name.into()
+	}
+}
+
+/// Set value of gauge metric.
+///
+/// If value is `Ok(None)` or `Err(_)`, metric would have default value.
+pub fn set_gauge_value<T: Default + Debug, V: Atomic<T = T>, E: Debug>(
+	gauge: &Gauge<V>,
+	value: Result<Option<T>, E>,
+) {
+	gauge.set(match value {
+		Ok(Some(value)) => {
+			tracing::trace!(
+				target: "bridge-metrics",
+				metric=?gauge.desc().first().map(|d| &d.fq_name),
+				?value,
+				"Updated value"
+			);
+			value
+		},
+		Ok(None) => {
+			tracing::warn!(
+				target: "bridge-metrics",
+				metric=?gauge.desc().first().map(|d| &d.fq_name),
+				"Failed to update: value is empty"
+			);
+			Default::default()
+		},
+		Err(error) => {
+			tracing::warn!(
+				target: "bridge-metrics",
+				?error,
+				metric=?gauge.desc().first().map(|d| &d.fq_name),
+				"Failed to update"
+			);
+			Default::default()
+		},
+	})
+}

pezbridges/relays/utils/src/metrics/float_json_value.rs

@@ -0,0 +1,147 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{
+	error::{self, Error},
+	metrics::{
+		metric_name, register, F64SharedRef, Gauge, Metric, PrometheusError, Registry,
+		StandaloneMetric, F64,
+	},
+};
+
+use async_std::sync::{Arc, RwLock};
+use async_trait::async_trait;
+use std::time::Duration;
+
+/// Value update interval.
+const UPDATE_INTERVAL: Duration = Duration::from_secs(300);
+
+/// Metric that represents float value received from HTTP service as float gauge.
+///
+/// The float value returned by the service is assumed to be normal (`f64::is_normal`
+/// should return `true`) and strictly positive.
+#[derive(Debug, Clone)]
+pub struct FloatJsonValueMetric {
+	url: String,
+	json_path: String,
+	metric: Gauge<F64>,
+	shared_value_ref: F64SharedRef,
+}
+
+impl FloatJsonValueMetric {
+	/// Create new metric instance with given name and help.
+	pub fn new(
+		url: String,
+		json_path: String,
+		name: String,
+		help: String,
+	) -> Result<Self, PrometheusError> {
+		let shared_value_ref = Arc::new(RwLock::new(None));
+		Ok(FloatJsonValueMetric {
+			url,
+			json_path,
+			metric: Gauge::new(metric_name(None, &name), help)?,
+			shared_value_ref,
+		})
+	}
+
+	/// Get shared reference to metric value.
+	pub fn shared_value_ref(&self) -> F64SharedRef {
+		self.shared_value_ref.clone()
+	}
+
+	/// Request value from HTTP service.
+	async fn request_value(&self) -> anyhow::Result<String> {
+		use isahc::{AsyncReadResponseExt, HttpClient, Request};
+
+		let request = Request::get(&self.url).header("Accept", "application/json").body(())?;
+		let raw_response = HttpClient::new()?.send_async(request).await?.text().await?;
+		Ok(raw_response)
+	}
+
+	/// Read value from HTTP service.
+	async fn read_value(&self) -> error::Result<f64> {
+		let raw_response = self.request_value().await.map_err(Error::FetchTokenPrice)?;
+		parse_service_response(&self.json_path, &raw_response)
+	}
+}
+
+impl Metric for FloatJsonValueMetric {
+	fn register(&self, registry: &Registry) -> Result<(), PrometheusError> {
+		register(self.metric.clone(), registry).map(drop)
+	}
+}
+
+#[async_trait]
+impl StandaloneMetric for FloatJsonValueMetric {
+	fn update_interval(&self) -> Duration {
+		UPDATE_INTERVAL
+	}
+
+	async fn update(&self) {
+		let value = self.read_value().await;
+		let maybe_ok = value.as_ref().ok().copied();
+		crate::metrics::set_gauge_value(&self.metric, value.map(Some));
+		*self.shared_value_ref.write().await = maybe_ok;
+	}
+}
+
+/// Parse HTTP service response.
+fn parse_service_response(json_path: &str, response: &str) -> error::Result<f64> {
+	let json =
+		serde_json::from_str(response).map_err(|err| Error::ParseHttp(err, response.to_owned()))?;
+
+	let mut selector = jsonpath_lib::selector(&json);
+	let maybe_selected_value =
+		selector(json_path).map_err(|err| Error::SelectResponseValue(err, response.to_owned()))?;
+	let selected_value = maybe_selected_value
+		.first()
+		.and_then(|v| v.as_f64())
+		.ok_or_else(|| Error::MissingResponseValue(response.to_owned()))?;
+	if !selected_value.is_normal() || selected_value < 0.0 {
+		return Err(Error::ParseFloat(selected_value));
+	}
+
+	Ok(selected_value)
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn parse_service_response_works() {
+		assert_eq!(
+			parse_service_response("$.kusama.usd", r#"{"kusama":{"usd":433.05}}"#).map_err(drop),
+			Ok(433.05),
+		);
+	}
+
+	#[test]
+	fn parse_service_response_rejects_negative_numbers() {
+		assert!(parse_service_response("$.kusama.usd", r#"{"kusama":{"usd":-433.05}}"#).is_err());
+	}
+
+	#[test]
+	fn parse_service_response_rejects_zero_numbers() {
+		assert!(parse_service_response("$.kusama.usd", r#"{"kusama":{"usd":0.0}}"#).is_err());
+	}
+
+	#[test]
+	fn parse_service_response_rejects_nan() {
+		assert!(parse_service_response("$.kusama.usd", r#"{"kusama":{"usd":NaN}}"#).is_err());
+	}
+}

pezbridges/relays/utils/src/metrics/global.rs

@@ -0,0 +1,118 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Global system-wide Prometheus metrics exposed by relays.
+
+use crate::metrics::{
+	metric_name, register, Gauge, GaugeVec, Metric, Opts, PrometheusError, Registry,
+	StandaloneMetric, F64, U64,
+};
+
+use async_std::sync::{Arc, Mutex};
+use async_trait::async_trait;
+use std::time::Duration;
+use sysinfo::{RefreshKind, System};
+
+/// Global metrics update interval.
+const UPDATE_INTERVAL: Duration = Duration::from_secs(10);
+
+/// Global Prometheus metrics.
+#[derive(Debug, Clone)]
+pub struct GlobalMetrics {
+	system: Arc<Mutex<System>>,
+	system_average_load: GaugeVec<F64>,
+	process_cpu_usage_percentage: Gauge<F64>,
+	process_memory_usage_bytes: Gauge<U64>,
+}
+
+impl GlobalMetrics {
+	/// Create and register global metrics.
+	pub fn new() -> Result<Self, PrometheusError> {
+		Ok(GlobalMetrics {
+			system: Arc::new(Mutex::new(System::new_with_specifics(RefreshKind::everything()))),
+			system_average_load: GaugeVec::new(
+				Opts::new(metric_name(None, "system_average_load"), "System load average"),
+				&["over"],
+			)?,
+			process_cpu_usage_percentage: Gauge::new(
+				metric_name(None, "process_cpu_usage_percentage"),
+				"Process CPU usage",
+			)?,
+			process_memory_usage_bytes: Gauge::new(
+				metric_name(None, "process_memory_usage_bytes"),
+				"Process memory (resident set size) usage",
+			)?,
+		})
+	}
+}
+
+impl Metric for GlobalMetrics {
+	fn register(&self, registry: &Registry) -> Result<(), PrometheusError> {
+		register(self.system_average_load.clone(), registry)?;
+		register(self.process_cpu_usage_percentage.clone(), registry)?;
+		register(self.process_memory_usage_bytes.clone(), registry)?;
+		Ok(())
+	}
+}
+
+#[async_trait]
+impl StandaloneMetric for GlobalMetrics {
+	async fn update(&self) {
+		// update system-wide metrics
+		let mut system = self.system.lock().await;
+		let load = sysinfo::System::load_average();
+		self.system_average_load.with_label_values(&["1min"]).set(load.one);
+		self.system_average_load.with_label_values(&["5min"]).set(load.five);
+		self.system_average_load.with_label_values(&["15min"]).set(load.fifteen);
+
+		// update process-related metrics
+		let pid = sysinfo::get_current_pid().expect(
+			"only fails where pid is unavailable (os=unknown || arch=wasm32);\
+				relay is not supposed to run in such MetricsParamss;\
+				qed",
+		);
+		let is_process_refreshed = system.refresh_process(pid);
+		match (is_process_refreshed, system.process(pid)) {
+			(true, Some(process_info)) => {
+				let cpu_usage = process_info.cpu_usage() as f64;
+				let memory_usage = process_info.memory() * 1024;
+				tracing::trace!(
+					target: "bridge-metrics",
+					?cpu_usage,
+					?memory_usage,
+					"Refreshed process metrics"
+				);
+
+				self.process_cpu_usage_percentage.set(if cpu_usage.is_finite() {
+					cpu_usage
+				} else {
+					0f64
+				});
+				self.process_memory_usage_bytes.set(memory_usage);
+			},
+			_ => {
+				tracing::warn!(
+					target: "bridge-metrics",
+					"Failed to refresh process information. Metrics may show obsolete values"
+				);
+			},
+		}
+	}
+
+	fn update_interval(&self) -> Duration {
+		UPDATE_INTERVAL
+	}
+}

pezbridges/relays/utils/src/relay_loop.rs

@@ -0,0 +1,262 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of Parity Bridges Common.
+
+// Parity Bridges Common is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Bridges Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Bridges Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{
+	error::Error,
+	metrics::{Metric, MetricsAddress, MetricsParams},
+	FailedClient, MaybeConnectionError,
+};
+
+use async_trait::async_trait;
+use prometheus_endpoint::{init_prometheus, Registry};
+use std::{fmt::Debug, future::Future, net::SocketAddr, time::Duration};
+
+/// Default pause between reconnect attempts.
+pub const RECONNECT_DELAY: Duration = Duration::from_secs(10);
+
+/// Basic blockchain client from relay perspective.
+#[async_trait]
+pub trait Client: 'static + Clone + Send + Sync {
+	/// Type of error these clients returns.
+	type Error: 'static + Debug + MaybeConnectionError + Send + Sync;
+
+	/// Try to reconnect to source node.
+	async fn reconnect(&mut self) -> Result<(), Self::Error>;
+
+	/// Try to reconnect to the source node in an infinite loop until it succeeds.
+	async fn reconnect_until_success(&mut self, delay: Duration) {
+		loop {
+			match self.reconnect().await {
+				Ok(()) => break,
+				Err(error) => {
+					tracing::warn!(
+						target: "bridge",
+						?error,
+						retry_as_secs=%delay.as_secs(),
+						"Failed to reconnect to client. Going to retry"
+					);
+
+					async_std::task::sleep(delay).await;
+				},
+			}
+		}
+	}
+}
+
+#[async_trait]
+impl Client for () {
+	type Error = crate::StringifiedMaybeConnectionError;
+
+	async fn reconnect(&mut self) -> Result<(), Self::Error> {
+		Ok(())
+	}
+}
+
+/// Returns generic loop that may be customized and started.
+pub fn relay_loop<SC, TC>(source_client: SC, target_client: TC) -> Loop<SC, TC, ()> {
+	Loop { reconnect_delay: RECONNECT_DELAY, source_client, target_client, loop_metric: None }
+}
+
+/// Returns generic relay loop metrics that may be customized and used in one or several relay
+/// loops.
+pub fn relay_metrics(params: MetricsParams) -> LoopMetrics<(), (), ()> {
+	LoopMetrics {
+		relay_loop: Loop {
+			reconnect_delay: RECONNECT_DELAY,
+			source_client: (),
+			target_client: (),
+			loop_metric: None,
+		},
+		address: params.address,
+		registry: params.registry,
+		loop_metric: None,
+	}
+}
+
+/// Generic relay loop.
+pub struct Loop<SC, TC, LM> {
+	reconnect_delay: Duration,
+	source_client: SC,
+	target_client: TC,
+	loop_metric: Option<LM>,
+}
+
+/// Relay loop metrics builder.
+pub struct LoopMetrics<SC, TC, LM> {
+	relay_loop: Loop<SC, TC, ()>,
+	address: Option<MetricsAddress>,
+	registry: Registry,
+	loop_metric: Option<LM>,
+}
+
+impl<SC, TC, LM> Loop<SC, TC, LM> {
+	/// Customize delay between reconnect attempts.
+	#[must_use]
+	pub fn reconnect_delay(mut self, reconnect_delay: Duration) -> Self {
+		self.reconnect_delay = reconnect_delay;
+		self
+	}
+
+	/// Start building loop metrics using given prefix.
+	pub fn with_metrics(self, params: MetricsParams) -> LoopMetrics<SC, TC, ()> {
+		LoopMetrics {
+			relay_loop: Loop {
+				reconnect_delay: self.reconnect_delay,
+				source_client: self.source_client,
+				target_client: self.target_client,
+				loop_metric: None,
+			},
+			address: params.address,
+			registry: params.registry,
+			loop_metric: None,
+		}
+	}
+
+	/// Run relay loop.
+	///
+	/// This function represents an outer loop, which in turn calls provided `run_loop` function to
+	/// do actual job. When `run_loop` returns, this outer loop reconnects to failed client (source,
+	/// target or both) and calls `run_loop` again.
+	pub async fn run<R, F>(mut self, loop_name: String, run_loop: R) -> Result<(), Error>
+	where
+		R: 'static + Send + Fn(SC, TC, Option<LM>) -> F,
+		F: 'static + Send + Future<Output = Result<(), FailedClient>>,
+		SC: 'static + Client,
+		TC: 'static + Client,
+		LM: 'static + Send + Clone,
+	{
+		let run_loop_task = async move {
+			crate::initialize::initialize_loop(loop_name);
+
+			loop {
+				let loop_metric = self.loop_metric.clone();
+				let future_result =
+					run_loop(self.source_client.clone(), self.target_client.clone(), loop_metric);
+				let result = future_result.await;
+
+				match result {
+					Ok(()) => break,
+					Err(failed_client) => {
+						tracing::debug!(target: "bridge", "Restarting relay loop");
+
+						reconnect_failed_client(
+							failed_client,
+							self.reconnect_delay,
+							&mut self.source_client,
+							&mut self.target_client,
+						)
+						.await
+					},
+				}
+			}
+			Ok(())
+		};
+
+		async_std::task::spawn(run_loop_task).await
+	}
+}
+
+impl<SC, TC, LM> LoopMetrics<SC, TC, LM> {
+	/// Add relay loop metrics.
+	///
+	/// Loop metrics will be passed to the loop callback.
+	pub fn loop_metric<NewLM: Metric>(
+		self,
+		metric: NewLM,
+	) -> Result<LoopMetrics<SC, TC, NewLM>, Error> {
+		metric.register(&self.registry)?;
+
+		Ok(LoopMetrics {
+			relay_loop: self.relay_loop,
+			address: self.address,
+			registry: self.registry,
+			loop_metric: Some(metric),
+		})
+	}
+
+	/// Convert into `MetricsParams` structure so that metrics registry may be extended later.
+	pub fn into_params(self) -> MetricsParams {
+		MetricsParams { address: self.address, registry: self.registry }
+	}
+
+	/// Expose metrics using address passed at creation.
+	///
+	/// If passed `address` is `None`, metrics are not exposed.
+	pub async fn expose(self) -> Result<Loop<SC, TC, LM>, Error> {
+		if let Some(address) = self.address {
+			let socket_addr = SocketAddr::new(
+				address
+					.host
+					.parse()
+					.map_err(|err| Error::ExposingMetricsInvalidHost(address.host.clone(), err))?,
+				address.port,
+			);
+
+			let registry = self.registry;
+			async_std::task::spawn(async move {
+				let runtime =
+					match tokio::runtime::Builder::new_current_thread().enable_all().build() {
+						Ok(runtime) => runtime,
+						Err(err) => {
+							tracing::trace!(
+								target: "bridge-metrics",
+								error=?err,
+								"Failed to create tokio runtime. Prometheus metrics are not available"
+							);
+							return;
+						},
+					};
+
+				runtime.block_on(async move {
+					tracing::trace!(
+						target: "bridge-metrics",
+						at=?socket_addr,
+						"Starting prometheus endpoint"
+					);
+					let result = init_prometheus(socket_addr, registry).await;
+					tracing::trace!(
+						target: "bridge-metrics",
+						?result,
+						"Prometheus endpoint has exited"
+					);
+				});
+			});
+		}
+
+		Ok(Loop {
+			reconnect_delay: self.relay_loop.reconnect_delay,
+			source_client: self.relay_loop.source_client,
+			target_client: self.relay_loop.target_client,
+			loop_metric: self.loop_metric,
+		})
+	}
+}
+
+/// Deal with the clients that have returned connection error.
+pub async fn reconnect_failed_client(
+	failed_client: FailedClient,
+	reconnect_delay: Duration,
+	source_client: &mut impl Client,
+	target_client: &mut impl Client,
+) {
+	if failed_client == FailedClient::Source || failed_client == FailedClient::Both {
+		source_client.reconnect_until_success(reconnect_delay).await;
+	}
+
+	if failed_client == FailedClient::Target || failed_client == FailedClient::Both {
+		target_client.reconnect_until_success(reconnect_delay).await;
+	}
+}