polkadot-node-subsystems: ChainApiBackend added + polkadot-debug image version fixed (#2411)

The out-dated version (bad tag) of [polkadot
image](https://github.com/paritytech/polkadot-sdk/blob/ede4a362622dfa69eb167eaa876246b1289f4b41/.gitlab/pipeline/zombienet/cumulus.yml#L31)
([docker
info](https://hub.docker.com/layers/paritypr/polkadot-debug/master/images/sha256:adb1658052cf671b50c90d5cece5c7a131efa1a95978249bd5cb85a5ad654f7a?context=explore))
was used. This PR fixes this.

See also:
https://github.com/paritytech/polkadot-sdk/pull/2411#issuecomment-1822632724

Also adds an abstraction that allows asynchronous backends to be passed to `ChainApiSubsystem`
---------

Co-authored-by: Sebastian Kunert <skunert49@gmail.com>

polkadot/node/core/chain-api/Cargo.toml

@@ -9,10 +9,9 @@ description = "The Chain API subsystem provides access to chain related utility
 [dependencies]
 futures = "0.3.21"
 gum = { package = "tracing-gum", path = "../../gum" }
-sp-blockchain = { path = "../../../../substrate/primitives/blockchain" }
-polkadot-primitives = { path = "../../../primitives" }
 polkadot-node-metrics = { path = "../../metrics" }
 polkadot-node-subsystem = { path = "../../subsystem" }
+polkadot-node-subsystem-types = { path = "../../subsystem-types" }
 sc-client-api = { path = "../../../../substrate/client/api" }
 sc-consensus-babe = { path = "../../../../substrate/client/consensus/babe" }
 
@@ -21,5 +20,7 @@ futures = { version = "0.3.21", features = ["thread-pool"] }
 maplit = "1.0.2"
 parity-scale-codec = "3.6.1"
 polkadot-node-primitives = { path = "../../primitives" }
+polkadot-primitives = { path = "../../../primitives" }
 polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
 sp-core = { path = "../../../../substrate/primitives/core" }
+sp-blockchain = { path = "../../../../substrate/primitives/blockchain" }

polkadot/node/core/chain-api/src/lib.rs

@@ -35,13 +35,13 @@ use std::sync::Arc;
 
 use futures::prelude::*;
 use sc_client_api::AuxStore;
-use sp_blockchain::HeaderBackend;
 
+use futures::stream::StreamExt;
 use polkadot_node_subsystem::{
 	messages::ChainApiMessage, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem,
 	SubsystemError, SubsystemResult,
 };
-use polkadot_primitives::Block;
+use polkadot_node_subsystem_types::ChainApiBackend;
 
 mod metrics;
 use self::metrics::Metrics;
@@ -67,7 +67,7 @@ impl<Client> ChainApiSubsystem<Client> {
 #[overseer::subsystem(ChainApi, error = SubsystemError, prefix = self::overseer)]
 impl<Client, Context> ChainApiSubsystem<Client>
 where
-	Client: HeaderBackend<Block> + AuxStore + 'static,
+	Client: ChainApiBackend + AuxStore + 'static,
 {
 	fn start(self, ctx: Context) -> SpawnedSubsystem {
 		let future = run::<Client, Context>(ctx, self)
@@ -83,7 +83,7 @@ async fn run<Client, Context>(
 	subsystem: ChainApiSubsystem<Client>,
 ) -> SubsystemResult<()>
 where
-	Client: HeaderBackend<Block> + AuxStore,
+	Client: ChainApiBackend + AuxStore,
 {
 	loop {
 		match ctx.recv().await? {
@@ -93,13 +93,15 @@ where
 			FromOrchestra::Communication { msg } => match msg {
 				ChainApiMessage::BlockNumber(hash, response_channel) => {
 					let _timer = subsystem.metrics.time_block_number();
-					let result = subsystem.client.number(hash).map_err(|e| e.to_string().into());
+					let result =
+						subsystem.client.number(hash).await.map_err(|e| e.to_string().into());
 					subsystem.metrics.on_request(result.is_ok());
 					let _ = response_channel.send(result);
 				},
 				ChainApiMessage::BlockHeader(hash, response_channel) => {
 					let _timer = subsystem.metrics.time_block_header();
-					let result = subsystem.client.header(hash).map_err(|e| e.to_string().into());
+					let result =
+						subsystem.client.header(hash).await.map_err(|e| e.to_string().into());
 					subsystem.metrics.on_request(result.is_ok());
 					let _ = response_channel.send(result);
 				},
@@ -113,46 +115,51 @@ where
 				ChainApiMessage::FinalizedBlockHash(number, response_channel) => {
 					let _timer = subsystem.metrics.time_finalized_block_hash();
 					// Note: we don't verify it's finalized
-					let result = subsystem.client.hash(number).map_err(|e| e.to_string().into());
+					let result =
+						subsystem.client.hash(number).await.map_err(|e| e.to_string().into());
 					subsystem.metrics.on_request(result.is_ok());
 					let _ = response_channel.send(result);
 				},
 				ChainApiMessage::FinalizedBlockNumber(response_channel) => {
 					let _timer = subsystem.metrics.time_finalized_block_number();
-					let result = subsystem.client.info().finalized_number;
-					// always succeeds
-					subsystem.metrics.on_request(true);
-					let _ = response_channel.send(Ok(result));
+					let result = subsystem
+						.client
+						.info()
+						.await
+						.map_err(|e| e.to_string().into())
+						.map(|info| info.finalized_number);
+					subsystem.metrics.on_request(result.is_ok());
+					let _ = response_channel.send(result);
 				},
 				ChainApiMessage::Ancestors { hash, k, response_channel } => {
 					let _timer = subsystem.metrics.time_ancestors();
 					gum::trace!(target: LOG_TARGET, hash=%hash, k=k, "ChainApiMessage::Ancestors");
 
-					let mut hash = hash;
+					let next_parent_stream = futures::stream::unfold(
+						(hash, subsystem.client.clone()),
+						|(hash, client)| async move {
+							let maybe_header = client.header(hash).await;
+							match maybe_header {
+								// propagate the error
+								Err(e) => {
+									let e = e.to_string().into();
+									Some((Err(e), (hash, client)))
+								},
+								// fewer than `k` ancestors are available
+								Ok(None) => None,
+								Ok(Some(header)) => {
+									// stop at the genesis header.
+									if header.number == 0 {
+										None
+									} else {
+										Some((Ok(header.parent_hash), (header.parent_hash, client)))
+									}
+								},
+							}
+						},
+					);
 
-					let next_parent = core::iter::from_fn(|| {
-						let maybe_header = subsystem.client.header(hash);
-						match maybe_header {
-							// propagate the error
-							Err(e) => {
-								let e = e.to_string().into();
-								Some(Err(e))
-							},
-							// fewer than `k` ancestors are available
-							Ok(None) => None,
-							Ok(Some(header)) => {
-								// stop at the genesis header.
-								if header.number == 0 {
-									None
-								} else {
-									hash = header.parent_hash;
-									Some(Ok(hash))
-								}
-							},
-						}
-					});
-
-					let result = next_parent.take(k).collect::<Result<Vec<_>, _>>();
+					let result = next_parent_stream.take(k).try_collect().await;
 					subsystem.metrics.on_request(result.is_ok());
 					let _ = response_channel.send(result);
 				},

polkadot/node/core/chain-api/src/tests.rs

@@ -22,7 +22,8 @@ use std::collections::BTreeMap;
 
 use polkadot_node_primitives::BlockWeight;
 use polkadot_node_subsystem_test_helpers::{make_subsystem_context, TestSubsystemContextHandle};
-use polkadot_primitives::{BlockNumber, Hash, Header};
+use polkadot_node_subsystem_types::ChainApiBackend;
+use polkadot_primitives::{Block, BlockNumber, Hash, Header};
 use sp_blockchain::Info as BlockInfo;
 use sp_core::testing::TaskExecutor;
 
@@ -110,7 +111,7 @@ fn last_key_value<K: Clone, V: Clone>(map: &BTreeMap<K, V>) -> (K, V) {
 	map.iter().last().map(|(k, v)| (k.clone(), v.clone())).unwrap()
 }
 
-impl HeaderBackend<Block> for TestClient {
+impl sp_blockchain::HeaderBackend<Block> for TestClient {
 	fn info(&self) -> BlockInfo<Block> {
 		let genesis_hash = self.blocks.iter().next().map(|(h, _)| *h).unwrap();
 		let (best_hash, best_number) = last_key_value(&self.blocks);
@@ -191,8 +192,8 @@ fn request_block_number() {
 		async move {
 			let zero = Hash::zero();
 			let test_cases = [
-				(TWO, client.number(TWO).unwrap()),
-				(zero, client.number(zero).unwrap()), // not here
+				(TWO, client.number(TWO).await.unwrap()),
+				(zero, client.number(zero).await.unwrap()), // not here
 			];
 			for (hash, expected) in &test_cases {
 				let (tx, rx) = oneshot::channel();
@@ -217,8 +218,10 @@ fn request_block_header() {
 	test_harness(|client, mut sender| {
 		async move {
 			const NOT_HERE: Hash = Hash::repeat_byte(0x5);
-			let test_cases =
-				[(TWO, client.header(TWO).unwrap()), (NOT_HERE, client.header(NOT_HERE).unwrap())];
+			let test_cases = [
+				(TWO, client.header(TWO).await.unwrap()),
+				(NOT_HERE, client.header(NOT_HERE).await.unwrap()),
+			];
 			for (hash, expected) in &test_cases {
 				let (tx, rx) = oneshot::channel();
 
@@ -270,8 +273,8 @@ fn request_finalized_hash() {
 	test_harness(|client, mut sender| {
 		async move {
 			let test_cases = [
-				(1, client.hash(1).unwrap()), // not here
-				(2, client.hash(2).unwrap()),
+				(1, client.hash(1).await.unwrap()), // not here
+				(2, client.hash(2).await.unwrap()),
 			];
 			for (number, expected) in &test_cases {
 				let (tx, rx) = oneshot::channel();
@@ -297,7 +300,7 @@ fn request_last_finalized_number() {
 		async move {
 			let (tx, rx) = oneshot::channel();
 
-			let expected = client.info().finalized_number;
+			let expected = client.info().await.unwrap().finalized_number;
 			sender
 				.send(FromOrchestra::Communication {
 					msg: ChainApiMessage::FinalizedBlockNumber(tx),

polkadot/node/overseer/src/lib.rs

@@ -87,8 +87,8 @@ use polkadot_node_subsystem_types::messages::{
 
 pub use polkadot_node_subsystem_types::{
 	errors::{SubsystemError, SubsystemResult},
-	jaeger, ActivatedLeaf, ActiveLeavesUpdate, OverseerSignal, RuntimeApiSubsystemClient,
-	UnpinHandle,
+	jaeger, ActivatedLeaf, ActiveLeavesUpdate, ChainApiBackend, OverseerSignal,
+	RuntimeApiSubsystemClient, UnpinHandle,
 };
 
 pub mod metrics;

polkadot/node/subsystem-types/Cargo.toml

@@ -17,6 +17,7 @@ polkadot-node-jaeger = { path = "../jaeger" }
 orchestra = { version = "0.3.3", default-features = false, features=["futures_channel"] }
 sc-network = { path = "../../../substrate/client/network" }
 sp-api = { path = "../../../substrate/primitives/api" }
+sp-blockchain = { path = "../../../substrate/primitives/blockchain" }
 sp-consensus-babe = { path = "../../../substrate/primitives/consensus/babe" }
 sp-authority-discovery = { path = "../../../substrate/primitives/authority-discovery" }
 sc-client-api = { path = "../../../substrate/client/api" }

polkadot/node/subsystem-types/src/lib.rs

@@ -40,7 +40,7 @@ pub mod errors;
 pub mod messages;
 
 mod runtime_client;
-pub use runtime_client::{DefaultSubsystemClient, RuntimeApiSubsystemClient};
+pub use runtime_client::{ChainApiBackend, DefaultSubsystemClient, RuntimeApiSubsystemClient};
 
 pub use jaeger::*;
 pub use polkadot_node_jaeger as jaeger;

polkadot/node/subsystem-types/src/runtime_client.rs

@@ -18,17 +18,70 @@ use async_trait::async_trait;
 use polkadot_primitives::{
 	async_backing, runtime_api::ParachainHost, slashing, vstaging, Block, BlockNumber,
 	CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState,
-	DisputeState, ExecutorParams, GroupRotationInfo, Hash, Id, InboundDownwardMessage,
+	DisputeState, ExecutorParams, GroupRotationInfo, Hash, Header, Id, InboundDownwardMessage,
 	InboundHrmpMessage, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement,
 	ScrapedOnChainVotes, SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash,
 	ValidatorId, ValidatorIndex, ValidatorSignature,
 };
+use sc_client_api::HeaderBackend;
 use sc_transaction_pool_api::OffchainTransactionPoolFactory;
-use sp_api::{ApiError, ApiExt, ProvideRuntimeApi};
+use sp_api::{ApiError, ApiExt, HeaderT, NumberFor, ProvideRuntimeApi};
 use sp_authority_discovery::AuthorityDiscoveryApi;
+use sp_blockchain::Info;
 use sp_consensus_babe::{BabeApi, Epoch};
 use std::{collections::BTreeMap, sync::Arc};
 
+/// Offers header utilities.
+///
+/// This is a async wrapper trait for ['HeaderBackend'] to be used with the
+/// `ChainApiSubsystem`.
+// This trait was introduced to suit the needs of collators. Depending on their operating mode, they
+// might not have a client of the relay chain that can supply a synchronous HeaderBackend
+// implementation.
+#[async_trait]
+pub trait ChainApiBackend: Send + Sync {
+	/// Get block header. Returns `None` if block is not found.
+	async fn header(&self, hash: Hash) -> sp_blockchain::Result<Option<Header>>;
+	/// Get blockchain info.
+	async fn info(&self) -> sp_blockchain::Result<Info<Block>>;
+	/// Get block number by hash. Returns `None` if the header is not in the chain.
+	async fn number(
+		&self,
+		hash: Hash,
+	) -> sp_blockchain::Result<Option<<Header as HeaderT>::Number>>;
+	/// Get block hash by number. Returns `None` if the header is not in the chain.
+	async fn hash(&self, number: NumberFor<Block>) -> sp_blockchain::Result<Option<Hash>>;
+}
+
+#[async_trait]
+impl<T> ChainApiBackend for T
+where
+	T: HeaderBackend<Block>,
+{
+	/// Get block header. Returns `None` if block is not found.
+	async fn header(&self, hash: Hash) -> sp_blockchain::Result<Option<Header>> {
+		HeaderBackend::header(self, hash)
+	}
+
+	/// Get blockchain info.
+	async fn info(&self) -> sp_blockchain::Result<Info<Block>> {
+		Ok(HeaderBackend::info(self))
+	}
+
+	/// Get block number by hash. Returns `None` if the header is not in the chain.
+	async fn number(
+		&self,
+		hash: Hash,
+	) -> sp_blockchain::Result<Option<<Header as HeaderT>::Number>> {
+		HeaderBackend::number(self, hash)
+	}
+
+	/// Get block hash by number. Returns `None` if the header is not in the chain.
+	async fn hash(&self, number: NumberFor<Block>) -> sp_blockchain::Result<Option<Hash>> {
+		HeaderBackend::hash(self, number)
+	}
+}
+
 /// Exposes all runtime calls that are used by the runtime API subsystem.
 #[async_trait]
 pub trait RuntimeApiSubsystemClient {